mcp-scraper 0.3.28 → 0.3.42

package/dist/chunk-OUZZNW3I.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.3.42";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-OUZZNW3I.js.map

package/dist/chunk-OUZZNW3I.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.3.42'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{chunk-H74L743B.js → chunk-RXJHXBGG.js}

@@ -6,7 +6,7 @@ import {
 import {
   finishHarvestAttempt,
   startHarvestAttempt
-} from "./chunk-TRNSXR46.js";
+} from "./chunk-DVRPXPYH.js";
 
 // src/api/rates.ts
 var MC_COSTS = {
@@ -23,6 +23,9 @@ var MC_COSTS = {
   maps_review: 100,
   fb_search: 600,
   fb_transcribe: 200,
+  google_ads_search: 600,
+  google_ads_intel: 200,
+  google_ads_transcribe: 200,
   instagram_profile: 400,
   instagram_media: 400,
   instagram_transcribe: 50,
@@ -122,6 +125,30 @@ var CREDIT_COST_CATALOG = [
     unit: "per search",
     notes: "Browser automation to search Facebook Ads Library by keyword."
   },
+  {
+    key: "google_ads_search",
+    label: "Google Ads Transparency search",
+    aliases: ["google_ads_search", "google ads search", "ads transparency search"],
+    credits: mcToCredits(MC_COSTS.google_ads_search),
+    unit: "per search",
+    notes: "Browser automation to find advertisers in Google Ads Transparency Center by domain or name."
+  },
+  {
+    key: "google_ads_intel",
+    label: "Google Ads Transparency advertiser intel",
+    aliases: ["google_ads_page_intel", "google ads intel", "ads transparency intel"],
+    credits: mcToCredits(MC_COSTS.google_ads_intel),
+    unit: "per call",
+    notes: "Lists and hydrates an advertiser's creatives with image URLs and video references."
+  },
+  {
+    key: "google_ads_transcribe",
+    label: "Google ad video transcription",
+    aliases: ["google_ads_transcribe", "google ad transcript"],
+    credits: mcToCredits(MC_COSTS.google_ads_transcribe),
+    unit: "per minute",
+    notes: "A hold is taken, then reconciled to actual video duration."
+  },
   {
     key: "fb_transcribe",
     label: "Facebook video / ad transcription",
@@ -215,7 +242,7 @@ function insufficientBalanceResponse(balanceMc, requiredMc) {
     topup_url: topupUrl
   };
 }
-var MEMORY_FREE_QUOTA_BYTES = 1e8 * 1;
+var MEMORY_FREE_QUOTA_BYTES = 1e7 * 1;
 var MEMORY_PLANS = {
   "price_1TnMSTS8aAcsk3TGBgiwuvqL": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGBgiwuvqL", interval: "month", monthly_usd: 19, quota_bytes: 5e9 },
   "price_1TnMSTS8aAcsk3TGWBVU2agY": { plan: "pro", label: "Pro", price_id: "price_1TnMSTS8aAcsk3TGWBVU2agY", interval: "year", monthly_usd: 19, quota_bytes: 5e9 }
@@ -223,6 +250,9 @@ var MEMORY_PLANS = {
 var MEMORY_PLAN_QUOTA = { free: MEMORY_FREE_QUOTA_BYTES, pro: 5e9, team: 5e10 };
 var MEMORY_MARGIN_MULTIPLE = 3;
 var MEMORY_PRO_COST_BUDGET_USD = 19 / MEMORY_MARGIN_MULTIPLE;
+var SCHEDULING_PLANS = {
+  "price_1ToXcHS8aAcsk3TGf3cW7zHx": { price_id: "price_1ToXcHS8aAcsk3TGf3cW7zHx", label: "Scheduled Actions", interval: "month", monthly_usd: 10, quota_per_period: 1e3 }
+};
 var LedgerOperation = {
   TOPUP: "topup",
   SUBSCRIPTION: "subscription",
@@ -253,6 +283,12 @@ var LedgerOperation = {
   FB_TRANSCRIBE: "fb_transcribe",
   FB_SEARCH_REFUND: "fb_search_refund",
   FB_TRANSCRIBE_REFUND: "fb_transcribe_refund",
+  GOOGLE_ADS_SEARCH: "google_ads_search",
+  GOOGLE_ADS_SEARCH_REFUND: "google_ads_search_refund",
+  GOOGLE_ADS_INTEL: "google_ads_intel",
+  GOOGLE_ADS_INTEL_REFUND: "google_ads_intel_refund",
+  GOOGLE_ADS_TRANSCRIBE: "google_ads_transcribe",
+  GOOGLE_ADS_TRANSCRIBE_REFUND: "google_ads_transcribe_refund",
   INSTAGRAM_PROFILE: "instagram_profile",
   INSTAGRAM_PROFILE_REFUND: "instagram_profile_refund",
   INSTAGRAM_MEDIA: "instagram_media",
@@ -419,10 +455,11 @@ export {
   insufficientBalanceResponse,
   MEMORY_PLANS,
   MEMORY_PLAN_QUOTA,
+  SCHEDULING_PLANS,
   LedgerOperation,
   classifyHarvestProblem,
   serializeHarvestProblem,
   harvestProblemResponse,
   createHarvestAttemptRecorder
 };
-//# sourceMappingURL=chunk-H74L743B.js.map
+//# sourceMappingURL=chunk-RXJHXBGG.js.map

package/dist/chunk-RXJHXBGG.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/rates.ts","../src/api/harvest-problems.ts","../src/api/harvest-attempt-events.ts"],"sourcesContent":["export const MC_COSTS = {\n  serp:              100,\n  paa:               200,\n  paa_base:        1_000,\n  page_scrape:       100,\n  url_map:           500,\n  yt_channel:        200,\n  yt_transcription:  200,\n  fb_ad:             200,\n  maps_search:       500,\n  maps_place:      3_000,\n  maps_review:       100,\n  fb_search:         600,\n  fb_transcribe:     200,\n  google_ads_search: 600,\n  google_ads_intel:  200,\n  google_ads_transcribe: 200,\n  instagram_profile: 400,\n  instagram_media:   400,\n  instagram_transcribe: 50,\n  browser_minute: 12_000,\n  reddit_thread:   2_000,\n} as const\n\nexport type McCostKey = keyof typeof MC_COSTS\n\nexport const MC_PER_BROWSER_MS = MC_COSTS.browser_minute / 60_000\n\nexport function browserActiveCostMc(activeMs: number): number {\n  return Math.round(activeMs * MC_PER_BROWSER_MS)\n}\n\nexport const BROWSER_OPEN_MIN_BALANCE_MC = 1_000\n\nexport const MC_PER_CREDIT = 1_000\n\nexport const CREDIT_COST_CATALOG: Array<{\n  key: McCostKey\n  label: string\n  aliases: string[]\n  credits: number\n  unit: string\n  notes?: string\n}> = [\n  {\n    key: 'serp',\n    label: 'SERP search',\n    aliases: ['search_serp', 'serp', 'google search', 'organic results'],\n    credits: mcToCredits(MC_COSTS.serp),\n    unit: 'per search',\n    notes: 'Returns AI Overview, PAA snippet, videos, forums, and local pack.',\n  },\n  {\n    key: 'paa',\n    label: 'PAA harvest',\n    aliases: ['harvest_paa', 'paa', 'people also ask', 'questions'],\n    credits: mcToCredits(MC_COSTS.paa),\n    unit: `per question (+${mcToCredits(MC_COSTS.paa_base)} credit base)`,\n    notes: `Full SERP feature extraction. Billed ${mcToCredits(MC_COSTS.paa_base)} credit base + ${mcToCredits(MC_COSTS.paa)} per question actually returned (unused estimate refunded).`,\n  },\n  {\n    key: 'page_scrape',\n    label: 'Page crawl / extract',\n    aliases: ['extract_url', 'extract_site', 'page scrape', 'url scrape', 'single page', 'site crawl'],\n    credits: mcToCredits(MC_COSTS.page_scrape),\n    unit: 'per page',\n    notes: 'Applies to both single-URL extraction and per-page site crawls.',\n  },\n  {\n    key: 'url_map',\n    label: 'Site URL mapping',\n    aliases: ['map_site_urls', 'url map', 'site map', 'crawl urls'],\n    credits: mcToCredits(MC_COSTS.url_map),\n    unit: 'per mapping operation',\n    notes: 'Flat rate for the full /map-urls call regardless of URL count discovered.',\n  },\n  {\n    key: 'yt_channel',\n    label: 'YouTube search / channel harvest',\n    aliases: ['youtube_harvest', 'youtube search', 'youtube channel', 'yt_channel'],\n    credits: mcToCredits(MC_COSTS.yt_channel),\n    unit: 'per call',\n  },\n  {\n    key: 'yt_transcription',\n    label: 'YouTube transcription',\n    aliases: ['youtube_transcribe', 'youtube transcript', 'transcription', 'yt_transcription'],\n    credits: mcToCredits(MC_COSTS.yt_transcription),\n    unit: 'per minute',\n    notes: 'A 5-minute hold is taken, then reconciled to actual video duration.',\n  },\n  {\n    key: 'fb_ad',\n    label: 'Facebook search / ad lookup',\n    aliases: ['facebook_page_intel', 'facebook_ad_search', 'facebook_ad', 'facebook ads', 'fb ads'],\n    credits: mcToCredits(MC_COSTS.fb_ad),\n    unit: 'per call',\n  },\n  {\n    key: 'maps_search',\n    label: 'Maps business search',\n    aliases: ['maps_search', 'google maps search', 'gmb search', 'gbp search', 'business profiles'],\n    credits: mcToCredits(MC_COSTS.maps_search),\n    unit: 'per search',\n    notes: 'Returns up to 50 Google Maps business/profile candidates. Use maps_place_intel to hydrate selected businesses.',\n  },\n  {\n    key: 'maps_place',\n    label: 'Maps business lookup',\n    aliases: ['maps_place_intel', 'google maps', 'maps place', 'place intel'],\n    credits: mcToCredits(MC_COSTS.maps_place),\n    unit: 'per business',\n    notes: 'Base lookup. Reviews billed separately per card at maps_review rate.',\n  },\n  {\n    key: 'maps_review',\n    label: 'Maps review',\n    aliases: ['maps_reviews', 'google reviews', 'review cards', 'reviews'],\n    credits: mcToCredits(MC_COSTS.maps_review),\n    unit: 'per review card',\n    notes: 'Charged after extraction when includeReviews is true.',\n  },\n  {\n    key: 'fb_search',\n    label: 'Facebook ad library search',\n    aliases: ['facebook_search', 'fb_search', 'fb ad search'],\n    credits: mcToCredits(MC_COSTS.fb_search),\n    unit: 'per search',\n    notes: 'Browser automation to search Facebook Ads Library by keyword.',\n  },\n  {\n    key: 'google_ads_search',\n    label: 'Google Ads Transparency search',\n    aliases: ['google_ads_search', 'google ads search', 'ads transparency search'],\n    credits: mcToCredits(MC_COSTS.google_ads_search),\n    unit: 'per search',\n    notes: 'Browser automation to find advertisers in Google Ads Transparency Center by domain or name.',\n  },\n  {\n    key: 'google_ads_intel',\n    label: 'Google Ads Transparency advertiser intel',\n    aliases: ['google_ads_page_intel', 'google ads intel', 'ads transparency intel'],\n    credits: mcToCredits(MC_COSTS.google_ads_intel),\n    unit: 'per call',\n    notes: 'Lists and hydrates an advertiser\\'s creatives with image URLs and video references.',\n  },\n  {\n    key: 'google_ads_transcribe',\n    label: 'Google ad video transcription',\n    aliases: ['google_ads_transcribe', 'google ad transcript'],\n    credits: mcToCredits(MC_COSTS.google_ads_transcribe),\n    unit: 'per minute',\n    notes: 'A hold is taken, then reconciled to actual video duration.',\n  },\n  {\n    key: 'fb_transcribe',\n    label: 'Facebook video / ad transcription',\n    aliases: ['facebook_transcribe', 'facebook_video_transcribe', 'fb_transcribe', 'fb ad transcript'],\n    credits: mcToCredits(MC_COSTS.fb_transcribe),\n    unit: 'per minute',\n    notes: 'A hold is taken, then reconciled to actual video duration.',\n  },\n  {\n    key: 'instagram_profile',\n    label: 'Instagram profile content discovery',\n    aliases: ['instagram_profile_content', 'instagram profile', 'instagram content list', 'ig profile'],\n    credits: mcToCredits(MC_COSTS.instagram_profile),\n    unit: 'per profile scan',\n    notes: 'Browser extraction of public Instagram profile grid links. Complete history may require a logged-in profile.',\n  },\n  {\n    key: 'instagram_media',\n    label: 'Instagram media download',\n    aliases: ['instagram_media_download', 'instagram reel download', 'instagram post download', 'ig media'],\n    credits: mcToCredits(MC_COSTS.instagram_media),\n    unit: 'per post or reel',\n    notes: 'Extracts post text, image URLs, and reel audio/video tracks, with local downloads when the server can write files.',\n  },\n  {\n    key: 'instagram_transcribe',\n    label: 'Instagram media transcription',\n    aliases: ['instagram transcript', 'instagram reel transcript', 'ig transcribe'],\n    credits: mcToCredits(MC_COSTS.instagram_transcribe),\n    unit: 'per call',\n    notes: 'Whisper transcription of selected Instagram audio/video media via fal.ai.',\n  },\n  {\n    key: 'browser_minute',\n    label: 'Interactive browser session',\n    aliases: ['browser_open', 'browser agent', 'browser_agent', 'live browser', 'browse', 'browser control', 'interactive browser'],\n    credits: mcToCredits(MC_COSTS.browser_minute),\n    unit: 'per minute of use',\n    notes: 'Metered per minute of use for the whole time the browser session is open. Close the session to stop the meter; abandoned sessions are auto-closed after a short idle window.',\n  },\n  {\n    key: 'reddit_thread',\n    label: 'Reddit thread + comments',\n    aliases: ['reddit', 'reddit_thread', 'reddit comments', 'subreddit', 'reddit post'],\n    credits: mcToCredits(MC_COSTS.reddit_thread),\n    unit: 'per thread',\n    notes: 'Captures a Reddit post and its comment tree. Charged on success; refunded if the thread cannot be retrieved.',\n  },\n]\n\nexport const CONCURRENCY_PRICE_ID = 'price_1Ta1NRS8aAcsk3TGwsRnYbix'\n\nexport interface SubscriptionTier { tier: string; label: string; price_id: string; monthly_usd: number; credits_mc: number; concurrency: number; intro_coupon: string | null }\n\nexport const SUBSCRIPTION_TIERS: Record<string, SubscriptionTier> = {\n  'price_1TmiHRS8aAcsk3TGwmSNfNIa': { tier: 'starter', label: 'Starter', price_id: 'price_1TmiHRS8aAcsk3TGwmSNfNIa', monthly_usd: 12, credits_mc: 8_000_000, concurrency: 3, intro_coupon: 'mcp-starter-1dollar-intro-12' },\n  'price_1Tmg1nS8aAcsk3TGe8zcnGTM': { tier: 'growth', label: 'Growth', price_id: 'price_1Tmg1nS8aAcsk3TGe8zcnGTM', monthly_usd: 100, credits_mc: 80_000_000, concurrency: 10, intro_coupon: null },\n  'price_1Tmg1nS8aAcsk3TGsZw34iXS': { tier: 'scale', label: 'Scale', price_id: 'price_1Tmg1nS8aAcsk3TGsZw34iXS', monthly_usd: 250, credits_mc: 240_000_000, concurrency: 20, intro_coupon: null },\n}\n\nexport const SUBSCRIPTION_TIER_BY_KEY: Record<string, SubscriptionTier> = Object.fromEntries(\n  Object.values(SUBSCRIPTION_TIERS).map(t => [t.tier, t]),\n)\nexport const CONCURRENCY_SLOT_PRICE_USD = 5\nexport const CONCURRENCY_SLOT_PRICE_CURRENCY = 'usd'\nexport const CONCURRENCY_SLOT_PRICE_INTERVAL = 'month'\nexport const CONCURRENCY_SLOT_PRICE_LABEL = '$5/month'\nexport const CONCURRENCY_SLOT_TERMINAL_COMMAND = 'npx -y -p mcp-scraper@latest mcp-scraper-cli billing concurrency checkout'\n\nexport function concurrencySlotBillingInfo() {\n  const billingUrl = process.env.CONCURRENCY_BILLING_URL ?? 'https://mcpscraper.dev/billing'\n  return {\n    product: 'Extra concurrency slot',\n    price_label: CONCURRENCY_SLOT_PRICE_LABEL,\n    unit_amount_usd: CONCURRENCY_SLOT_PRICE_USD,\n    currency: CONCURRENCY_SLOT_PRICE_CURRENCY,\n    interval: CONCURRENCY_SLOT_PRICE_INTERVAL,\n    billing_url: billingUrl,\n    terminal_command: CONCURRENCY_SLOT_TERMINAL_COMMAND,\n    terminal_command_with_api_key_env: `MCP_SCRAPER_API_KEY=sk_live_your_key ${CONCURRENCY_SLOT_TERMINAL_COMMAND}`,\n  }\n}\n\nexport const FREE_SIGNUP_MC = 100_000\nexport const FREE_MONTHLY_REFRESH_MC = 250_000\n\nexport const BALANCE_PRICE_IDS: Record<string, number> = {\n  'price_1TZx6rS8aAcsk3TGNMc1Vgpo': 11_000_000,\n  'price_1TZx6sS8aAcsk3TGxgqB7khO': 27_500_000,\n  'price_1TZx6tS8aAcsk3TG8PnJqHlG': 60_500_000,\n  'price_1TZx6tS8aAcsk3TGNgRMpy0e': 121_000_000,\n}\n\nexport const BALANCE_PACK_LABELS: Record<string, string> = {\n  'price_1TZx6rS8aAcsk3TGNMc1Vgpo': '$10',\n  'price_1TZx6sS8aAcsk3TGxgqB7khO': '$25',\n  'price_1TZx6tS8aAcsk3TG8PnJqHlG': '$50',\n  'price_1TZx6tS8aAcsk3TGNgRMpy0e': '$100',\n}\n\nexport function mcToCredits(mc: number): number {\n  return mc / MC_PER_CREDIT\n}\n\nexport function insufficientBalanceResponse(balanceMc: number, requiredMc: number) {\n  const topupUrl = process.env.TOPUP_URL ?? 'https://mcpscraper.dev/billing'\n  const balanceCredits = mcToCredits(balanceMc)\n  const requiredCredits = mcToCredits(requiredMc)\n  return {\n    error: 'insufficient_balance',\n    error_code: 'insufficient_balance' as const,\n    message: `Insufficient credits. Balance: ${balanceCredits} credits. This call requires ${requiredCredits} credits. Top up at ${topupUrl}`,\n    balance_credits: balanceCredits,\n    required_credits: requiredCredits,\n    topup_url: topupUrl,\n  }\n}\n\nexport interface MemoryPlan {\n  plan: 'pro' | 'team'\n  label: string\n  price_id: string\n  interval: 'month' | 'year'\n  monthly_usd: number\n  quota_bytes: number\n}\n\nexport const MEMORY_FREE_QUOTA_BYTES = 0.01e9 * 1\n\nexport const MEMORY_PLANS: Record<string, MemoryPlan> = {\n  'price_1TnMSTS8aAcsk3TGBgiwuvqL': { plan: 'pro', label: 'Pro', price_id: 'price_1TnMSTS8aAcsk3TGBgiwuvqL', interval: 'month', monthly_usd: 19, quota_bytes: 5e9 },\n  'price_1TnMSTS8aAcsk3TGWBVU2agY': { plan: 'pro', label: 'Pro', price_id: 'price_1TnMSTS8aAcsk3TGWBVU2agY', interval: 'year', monthly_usd: 19, quota_bytes: 5e9 },\n}\n\nexport const MEMORY_PLAN_QUOTA: Record<string, number> = { free: MEMORY_FREE_QUOTA_BYTES, pro: 5e9, team: 50e9 }\n\nexport const MEMORY_MARGIN_MULTIPLE = 3\nexport const MEMORY_FREE_COST_CAP_USD = 1.0\nexport const MEMORY_PRO_COST_BUDGET_USD = 19 / MEMORY_MARGIN_MULTIPLE\n\nexport interface SchedulingPlan {\n  price_id: string\n  label: string\n  interval: 'month'\n  monthly_usd: number\n  quota_per_period: number\n}\n\nexport const SCHEDULING_PLANS: Record<string, SchedulingPlan> = {\n  'price_1ToXcHS8aAcsk3TGf3cW7zHx': { price_id: 'price_1ToXcHS8aAcsk3TGf3cW7zHx', label: 'Scheduled Actions', interval: 'month', monthly_usd: 10, quota_per_period: 1000 },\n}\n\nexport const LedgerOperation = {\n  TOPUP:                 'topup',\n  SUBSCRIPTION:          'subscription',\n  SIGNUP_GRANT:          'signup_grant',\n  MONTHLY_REFRESH:       'monthly_free_refresh',\n  PAA:                   'paa',\n  PAA_REFUND:            'paa_refund',\n  SERP:                  'serp',\n  REFUND:                'refund',\n  TRANSCRIPTION:         'transcription',\n  TRANSCRIPTION_HOLD:    'transcription_hold',\n  TRANSCRIPTION_REFUND:  'transcription_refund',\n  YT_CHANNEL:            'yt_channel',\n  FB_AD:                 'fb_ad',\n  MAPS_SEARCH:           'maps_search',\n  MAPS_PLACE:            'maps_place',\n  MAPS_REVIEW:           'maps_review',\n  MAPS_REVIEW_REFUND:    'maps_review_refund',\n  EXTRACT_SITE:          'extract_site',\n  EXTRACT_SITE_REFUND:   'extract_site_refund',\n  EXTRACT_URL:           'page_scrape',\n  URL_MAP:               'url_map',\n  EXTRACT_SITE_HOLD:     'extract_site_hold',\n  YT_CHANNEL_REFUND:     'yt_channel_refund',\n  FB_AD_REFUND:          'fb_ad_refund',\n  URL_MAP_REFUND:        'url_map_refund',\n  FB_SEARCH:             'fb_search',\n  FB_TRANSCRIBE:         'fb_transcribe',\n  FB_SEARCH_REFUND:      'fb_search_refund',\n  FB_TRANSCRIBE_REFUND:  'fb_transcribe_refund',\n  GOOGLE_ADS_SEARCH:     'google_ads_search',\n  GOOGLE_ADS_SEARCH_REFUND: 'google_ads_search_refund',\n  GOOGLE_ADS_INTEL:      'google_ads_intel',\n  GOOGLE_ADS_INTEL_REFUND: 'google_ads_intel_refund',\n  GOOGLE_ADS_TRANSCRIBE: 'google_ads_transcribe',\n  GOOGLE_ADS_TRANSCRIBE_REFUND: 'google_ads_transcribe_refund',\n  INSTAGRAM_PROFILE:     'instagram_profile',\n  INSTAGRAM_PROFILE_REFUND: 'instagram_profile_refund',\n  INSTAGRAM_MEDIA:       'instagram_media',\n  INSTAGRAM_MEDIA_REFUND:'instagram_media_refund',\n  INSTAGRAM_TRANSCRIBE:  'instagram_transcribe',\n  INSTAGRAM_TRANSCRIBE_REFUND: 'instagram_transcribe_refund',\n  BROWSER_SESSION:       'browser_session',\n  REDDIT_THREAD:         'reddit_thread',\n  REDDIT_THREAD_REFUND:  'reddit_thread_refund',\n} as const\n\nexport type LedgerOperation = typeof LedgerOperation[keyof typeof LedgerOperation]\n","import { CaptchaError, LocationMismatchError, RequestAbortedError } from '../errors.js'\n\nexport type HarvestProblemCode =\n  | 'request_aborted'\n  | 'captcha_exhausted'\n  | 'location_mismatch'\n  | 'proxy_tunnel_failed'\n  | 'proxy_unavailable'\n  | 'harvest_timeout'\n  | 'extraction_failed'\n\nexport interface HarvestProblem {\n  error_code: HarvestProblemCode\n  error_type: string\n  message: string\n  retryable: boolean\n  httpStatus: number\n  terminalStatus: 'cancelled' | 'failed'\n}\n\nfunction errorMessage(err: unknown): string {\n  return err instanceof Error ? err.message : String(err)\n}\n\nfunction looksLikeTimeout(err: unknown, message: string): boolean {\n  if (err instanceof DOMException && (err.name === 'TimeoutError' || err.name === 'AbortError')) return true\n  return /timeout|timed out|Timeout \\d+ms exceeded|deadline/i.test(message)\n}\n\nfunction looksLikeCaptcha(message: string): boolean {\n  return /captcha|recaptcha|unusual traffic|google\\.com\\/sorry|blocked/i.test(message)\n}\n\nfunction looksLikeProxyTunnelFailure(message: string): boolean {\n  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message)\n}\n\nfunction looksLikeProxyUnavailable(message: string): boolean {\n  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message)\n}\n\nexport function classifyHarvestProblem(err: unknown): HarvestProblem {\n  const message = errorMessage(err)\n\n  if (err instanceof RequestAbortedError) {\n    return {\n      error_code: 'request_aborted',\n      error_type: 'request_aborted',\n      message,\n      retryable: true,\n      httpStatus: 408,\n      terminalStatus: 'cancelled',\n    }\n  }\n\n  if (err instanceof CaptchaError || looksLikeCaptcha(message)) {\n    return {\n      error_code: 'captcha_exhausted',\n      error_type: 'captcha',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (err instanceof LocationMismatchError) {\n    return {\n      error_code: 'location_mismatch',\n      error_type: 'location_mismatch',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyTunnelFailure(message)) {\n    return {\n      error_code: 'proxy_tunnel_failed',\n      error_type: 'proxy_tunnel_failed',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyUnavailable(message)) {\n    return {\n      error_code: 'proxy_unavailable',\n      error_type: 'proxy_unavailable',\n      message,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeTimeout(err, message)) {\n    return {\n      error_code: 'harvest_timeout',\n      error_type: 'timeout',\n      message,\n      retryable: true,\n      httpStatus: 504,\n      terminalStatus: 'failed',\n    }\n  }\n\n  return {\n    error_code: 'extraction_failed',\n    error_type: 'extraction',\n    message,\n    retryable: false,\n    httpStatus: 500,\n    terminalStatus: 'failed',\n  }\n}\n\nexport function serializeHarvestProblem(problem: HarvestProblem): string {\n  return JSON.stringify({\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    message: problem.message,\n    retryable: problem.retryable,\n  })\n}\n\nexport function harvestProblemResponse(problem: HarvestProblem): {\n  error: string\n  error_code: HarvestProblemCode\n  error_type: string\n  retryable: boolean\n} {\n  return {\n    error: problem.message,\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    retryable: problem.retryable,\n  }\n}\n","import type { HarvestAttemptLogEvent } from '../harvest.js'\nimport { finishHarvestAttempt, startHarvestAttempt } from './db.js'\n\nexport function createHarvestAttemptRecorder(jobId: string, userId: number | bigint) {\n  return async (event: HarvestAttemptLogEvent): Promise<void> => {\n    if (event.type === 'started') {\n      await startHarvestAttempt({\n        jobId,\n        userId,\n        attemptNumber: event.attemptNumber,\n        maxAttempts: event.maxAttempts,\n        query: event.query,\n        location: event.location,\n        maxQuestions: event.maxQuestions,\n        startedAt: event.startedAt,\n      })\n      return\n    }\n\n    await finishHarvestAttempt({\n      jobId,\n      attemptNumber: event.attemptNumber,\n      outcome: event.outcome,\n      kernelSessionId: event.kernelSessionId,\n      questionCount: event.questionCount,\n      durationMs: event.durationMs,\n      error: event.error,\n      willRetry: event.willRetry,\n      kernelDeleteStarted: event.cleanup.kernelDeleteStarted,\n      kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,\n      kernelDeleteError: event.cleanup.kernelDeleteError,\n      browserCloseSucceeded: event.cleanup.browserCloseSucceeded,\n      browserCloseError: event.cleanup.browserCloseError,\n      debug: event.debug,\n      completedAt: event.completedAt,\n    })\n  }\n}\n"],"mappings":";;;;;;;;;;;AAAO,IAAM,WAAW;AAAA,EACtB,MAAmB;AAAA,EACnB,KAAmB;AAAA,EACnB,UAAiB;AAAA,EACjB,aAAmB;AAAA,EACnB,SAAmB;AAAA,EACnB,YAAmB;AAAA,EACnB,kBAAmB;AAAA,EACnB,OAAmB;AAAA,EACnB,aAAmB;AAAA,EACnB,YAAiB;AAAA,EACjB,aAAmB;AAAA,EACnB,WAAmB;AAAA,EACnB,eAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,kBAAmB;AAAA,EACnB,uBAAuB;AAAA,EACvB,mBAAmB;AAAA,EACnB,iBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,gBAAgB;AAAA,EAChB,eAAiB;AACnB;AAIO,IAAM,oBAAoB,SAAS,iBAAiB;AAEpD,SAAS,oBAAoB,UAA0B;AAC5D,SAAO,KAAK,MAAM,WAAW,iBAAiB;AAChD;AAEO,IAAM,8BAA8B;AAEpC,IAAM,gBAAgB;AAEtB,IAAM,sBAOR;AAAA,EACH;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,QAAQ,iBAAiB,iBAAiB;AAAA,IACnE,SAAS,YAAY,SAAS,IAAI;AAAA,IAClC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,OAAO,mBAAmB,WAAW;AAAA,IAC9D,SAAS,YAAY,SAAS,GAAG;AAAA,IACjC,MAAM,kBAAkB,YAAY,SAAS,QAAQ,CAAC;AAAA,IACtD,OAAO,wCAAwC,YAAY,SAAS,QAAQ,CAAC,kBAAkB,YAAY,SAAS,GAAG,CAAC;AAAA,EAC1H;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,gBAAgB,eAAe,cAAc,eAAe,YAAY;AAAA,IACjG,SAAS,YAAY,SAAS,WAAW;AAAA,IACzC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,iBAAiB,WAAW,YAAY,YAAY;AAAA,IAC9D,SAAS,YAAY,SAAS,OAAO;AAAA,IACrC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,mBAAmB,kBAAkB,mBAAmB,YAAY;AAAA,IAC9E,SAAS,YAAY,SAAS,UAAU;AAAA,IACxC,MAAM;AAAA,EACR;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,sBAAsB,sBAAsB,iBAAiB,kBAAkB;AAAA,IACzF,SAAS,YAAY,SAAS,gBAAgB;AAAA,IAC9C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,uBAAuB,sBAAsB,eAAe,gBAAgB,QAAQ;AAAA,IAC9F,SAAS,YAAY,SAAS,KAAK;AAAA,IACnC,MAAM;AAAA,EACR;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,eAAe,sBAAsB,cAAc,cAAc,mBAAmB;AAAA,IAC9F,SAAS,YAAY,SAAS,WAAW;AAAA,IACzC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,oBAAoB,eAAe,cAAc,aAAa;AAAA,IACxE,SAAS,YAAY,SAAS,UAAU;AAAA,IACxC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,gBAAgB,kBAAkB,gBAAgB,SAAS;AAAA,IACrE,SAAS,YAAY,SAAS,WAAW;AAAA,IACzC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,mBAAmB,aAAa,cAAc;AAAA,IACxD,SAAS,YAAY,SAAS,SAAS;AAAA,IACvC,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,qBAAqB,qBAAqB,yBAAyB;AAAA,IAC7E,SAAS,YAAY,SAAS,iBAAiB;AAAA,IAC/C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,yBAAyB,oBAAoB,wBAAwB;AAAA,IAC/E,SAAS,YAAY,SAAS,gBAAgB;AAAA,IAC9C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,yBAAyB,sBAAsB;AAAA,IACzD,SAAS,YAAY,SAAS,qBAAqB;AAAA,IACnD,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,uBAAuB,6BAA6B,iBAAiB,kBAAkB;AAAA,IACjG,SAAS,YAAY,SAAS,aAAa;AAAA,IAC3C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,6BAA6B,qBAAqB,0BAA0B,YAAY;AAAA,IAClG,SAAS,YAAY,SAAS,iBAAiB;AAAA,IAC/C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,4BAA4B,2BAA2B,2BAA2B,UAAU;AAAA,IACtG,SAAS,YAAY,SAAS,eAAe;AAAA,IAC7C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,wBAAwB,6BAA6B,eAAe;AAAA,IAC9E,SAAS,YAAY,SAAS,oBAAoB;AAAA,IAClD,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,gBAAgB,iBAAiB,iBAAiB,gBAAgB,UAAU,mBAAmB,qBAAqB;AAAA,IAC9H,SAAS,YAAY,SAAS,cAAc;AAAA,IAC5C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AAAA,EACA;AAAA,IACE,KAAK;AAAA,IACL,OAAO;AAAA,IACP,SAAS,CAAC,UAAU,iBAAiB,mBAAmB,aAAa,aAAa;AAAA,IAClF,SAAS,YAAY,SAAS,aAAa;AAAA,IAC3C,MAAM;AAAA,IACN,OAAO;AAAA,EACT;AACF;AAEO,IAAM,uBAAuB;AAI7B,IAAM,qBAAuD;AAAA,EAClE,kCAAkC,EAAE,MAAM,WAAW,OAAO,WAAW,UAAU,kCAAkC,aAAa,IAAI,YAAY,KAAW,aAAa,GAAG,cAAc,+BAA+B;AAAA,EACxN,kCAAkC,EAAE,MAAM,UAAU,OAAO,UAAU,UAAU,kCAAkC,aAAa,KAAK,YAAY,KAAY,aAAa,IAAI,cAAc,KAAK;AAAA,EAC/L,kCAAkC,EAAE,MAAM,SAAS,OAAO,SAAS,UAAU,kCAAkC,aAAa,KAAK,YAAY,MAAa,aAAa,IAAI,cAAc,KAAK;AAChM;AAEO,IAAM,2BAA6D,OAAO;AAAA,EAC/E,OAAO,OAAO,kBAAkB,EAAE,IAAI,OAAK,CAAC,EAAE,MAAM,CAAC,CAAC;AACxD;AACO,IAAM,6BAA6B;AACnC,IAAM,kCAAkC;AACxC,IAAM,kCAAkC;AACxC,IAAM,+BAA+B;AACrC,IAAM,oCAAoC;AAE1C,SAAS,6BAA6B;AAC3C,QAAM,aAAa,QAAQ,IAAI,2BAA2B;AAC1D,SAAO;AAAA,IACL,SAAS;AAAA,IACT,aAAa;AAAA,IACb,iBAAiB;AAAA,IACjB,UAAU;AAAA,IACV,UAAU;AAAA,IACV,aAAa;AAAA,IACb,kBAAkB;AAAA,IAClB,mCAAmC,wCAAwC,iCAAiC;AAAA,EAC9G;AACF;AAEO,IAAM,iBAAiB;AAiBvB,SAAS,YAAY,IAAoB;AAC9C,SAAO,KAAK;AACd;AAEO,SAAS,4BAA4B,WAAmB,YAAoB;AACjF,QAAM,WAAW,QAAQ,IAAI,aAAa;AAC1C,QAAM,iBAAiB,YAAY,SAAS;AAC5C,QAAM,kBAAkB,YAAY,UAAU;AAC9C,SAAO;AAAA,IACL,OAAO;AAAA,IACP,YAAY;AAAA,IACZ,SAAS,kCAAkC,cAAc,gCAAgC,eAAe,uBAAuB,QAAQ;AAAA,IACvI,iBAAiB;AAAA,IACjB,kBAAkB;AAAA,IAClB,WAAW;AAAA,EACb;AACF;AAWO,IAAM,0BAA0B,MAAS;AAEzC,IAAM,eAA2C;AAAA,EACtD,kCAAkC,EAAE,MAAM,OAAO,OAAO,OAAO,UAAU,kCAAkC,UAAU,SAAS,aAAa,IAAI,aAAa,IAAI;AAAA,EAChK,kCAAkC,EAAE,MAAM,OAAO,OAAO,OAAO,UAAU,kCAAkC,UAAU,QAAQ,aAAa,IAAI,aAAa,IAAI;AACjK;AAEO,IAAM,oBAA4C,EAAE,MAAM,yBAAyB,KAAK,KAAK,MAAM,KAAK;AAExG,IAAM,yBAAyB;AAE/B,IAAM,6BAA6B,KAAK;AAUxC,IAAM,mBAAmD;AAAA,EAC9D,kCAAkC,EAAE,UAAU,kCAAkC,OAAO,qBAAqB,UAAU,SAAS,aAAa,IAAI,kBAAkB,IAAK;AACzK;AAEO,IAAM,kBAAkB;AAAA,EAC7B,OAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,iBAAuB;AAAA,EACvB,KAAuB;AAAA,EACvB,YAAuB;AAAA,EACvB,MAAuB;AAAA,EACvB,QAAuB;AAAA,EACvB,eAAuB;AAAA,EACvB,oBAAuB;AAAA,EACvB,sBAAuB;AAAA,EACvB,YAAuB;AAAA,EACvB,OAAuB;AAAA,EACvB,aAAuB;AAAA,EACvB,YAAuB;AAAA,EACvB,aAAuB;AAAA,EACvB,oBAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,qBAAuB;AAAA,EACvB,aAAuB;AAAA,EACvB,SAAuB;AAAA,EACvB,mBAAuB;AAAA,EACvB,mBAAuB;AAAA,EACvB,cAAuB;AAAA,EACvB,gBAAuB;AAAA,EACvB,WAAuB;AAAA,EACvB,eAAuB;AAAA,EACvB,kBAAuB;AAAA,EACvB,sBAAuB;AAAA,EACvB,mBAAuB;AAAA,EACvB,0BAA0B;AAAA,EAC1B,kBAAuB;AAAA,EACvB,yBAAyB;AAAA,EACzB,uBAAuB;AAAA,EACvB,8BAA8B;AAAA,EAC9B,mBAAuB;AAAA,EACvB,0BAA0B;AAAA,EAC1B,iBAAuB;AAAA,EACvB,wBAAuB;AAAA,EACvB,sBAAuB;AAAA,EACvB,6BAA6B;AAAA,EAC7B,iBAAuB;AAAA,EACvB,eAAuB;AAAA,EACvB,sBAAuB;AACzB;;;AC3UA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,iBAAiB,KAAc,SAA0B;AAChE,MAAI,eAAe,iBAAiB,IAAI,SAAS,kBAAkB,IAAI,SAAS,cAAe,QAAO;AACtG,SAAO,qDAAqD,KAAK,OAAO;AAC1E;AAEA,SAAS,iBAAiB,SAA0B;AAClD,SAAO,gEAAgE,KAAK,OAAO;AACrF;AAEA,SAAS,4BAA4B,SAA0B;AAC7D,SAAO,gKAAgK,KAAK,OAAO;AACrL;AAEA,SAAS,0BAA0B,SAA0B;AAC3D,SAAO,4GAA4G,KAAK,OAAO;AACjI;AAEO,SAAS,uBAAuB,KAA8B;AACnE,QAAM,UAAU,aAAa,GAAG;AAEhC,MAAI,eAAe,qBAAqB;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,gBAAgB,iBAAiB,OAAO,GAAG;AAC5D,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,uBAAuB;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,4BAA4B,OAAO,GAAG;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,0BAA0B,OAAO,GAAG;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,iBAAiB,KAAK,OAAO,GAAG;AAClC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ;AAAA,MACA,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ;AAAA,IACA,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,gBAAgB;AAAA,EAClB;AACF;AAEO,SAAS,wBAAwB,SAAiC;AACvE,SAAO,KAAK,UAAU;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,SAAS,QAAQ;AAAA,IACjB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACH;AAEO,SAAS,uBAAuB,SAKrC;AACA,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB;AACF;;;AC1IO,SAAS,6BAA6B,OAAe,QAAyB;AACnF,SAAO,OAAO,UAAiD;AAC7D,QAAI,MAAM,SAAS,WAAW;AAC5B,YAAM,oBAAoB;AAAA,QACxB;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,aAAa,MAAM;AAAA,QACnB,OAAO,MAAM;AAAA,QACb,UAAU,MAAM;AAAA,QAChB,cAAc,MAAM;AAAA,QACpB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA,eAAe,MAAM;AAAA,MACrB,SAAS,MAAM;AAAA,MACf,iBAAiB,MAAM;AAAA,MACvB,eAAe,MAAM;AAAA,MACrB,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,WAAW,MAAM;AAAA,MACjB,qBAAqB,MAAM,QAAQ;AAAA,MACnC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,OAAO,MAAM;AAAA,MACb,aAAa,MAAM;AAAA,IACrB,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/{db-FP2ABUU4.js → db-LYQENFPW.js}

@@ -97,7 +97,7 @@ import {
   stripeEventAlreadyProcessed,
   updateKpoJobState,
   verifyPassword
-} from "./chunk-TRNSXR46.js";
+} from "./chunk-DVRPXPYH.js";
 export {
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
@@ -198,4 +198,4 @@ export {
   updateKpoJobState,
   verifyPassword
 };
-//# sourceMappingURL=db-FP2ABUU4.js.map
+//# sourceMappingURL=db-LYQENFPW.js.map

package/dist/index.cjs

@@ -1597,6 +1597,13 @@ var PAAExtractor = class {
   }
   driver;
   reporter;
+  completeness = {
+    paaWithoutAnswer: 0,
+    paaWithoutSource: 0,
+    paaAnswersRecovered: 0,
+    aioShareCaptured: null
+  };
+  aioShareUrlEarly = null;
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -1725,6 +1732,7 @@ var PAAExtractor = class {
       }
     }
     const itemMap = new Map((await this.extractVisibleItems(page)).map((i) => [i.question, i]));
+    await this.fillIncompleteItems(page, orderedQs, itemMap, options);
     for (const q of orderedQs) {
       if (results.length >= options.maxQuestions) break;
       const key = this.normalizeQuestion(q);
@@ -1738,8 +1746,52 @@ var PAAExtractor = class {
         results.push(this.toFlatRow({ question: q, answer: void 0, sourceTitle: void 0, sourceSite: void 0, sourceCite: void 0 }, 1, null, options.query));
       }
     }
+    this.completeness.paaWithoutAnswer = results.filter((r) => !r.answer).length;
+    this.completeness.paaWithoutSource = results.filter((r) => !r.source_title && !r.source_site && !r.source_cite).length;
     return results;
   }
+  async fillIncompleteItems(page, orderedQs, itemMap, options) {
+    const kept = new Set(orderedQs.slice(0, options.maxQuestions));
+    const missingBefore = new Set([...kept].filter((q) => !itemMap.get(q)?.answer));
+    if (missingBefore.size === 0) return;
+    for (let round = 0; round < 2; round++) {
+      if (options.softDeadlineMs && Date.now() >= options.softDeadlineMs) break;
+      const stillMissing = [...missingBefore].filter((q) => !itemMap.get(q)?.answer);
+      if (stillMissing.length === 0) break;
+      const reExpanded = await this.reExpandCollapsedItems(page).catch(() => false);
+      await page.waitForTimeout(reExpanded ? 900 : 500);
+      let improved = false;
+      for (const item of await this.extractVisibleItems(page)) {
+        if (!kept.has(item.question)) continue;
+        const existing = itemMap.get(item.question);
+        if (!item.answer || existing?.answer) continue;
+        itemMap.set(item.question, {
+          ...item,
+          sourceTitle: item.sourceTitle ?? existing?.sourceTitle,
+          sourceSite: item.sourceSite ?? existing?.sourceSite,
+          sourceCite: item.sourceCite ?? existing?.sourceCite
+        });
+        improved = true;
+      }
+      if (!improved) break;
+    }
+    this.completeness.paaAnswersRecovered = [...missingBefore].filter((q) => itemMap.get(q)?.answer).length;
+  }
+  async reExpandCollapsedItems(page) {
+    const unexpandedSel = `${PAASelectors.item}:not(.${PAASelectors.expandedClass}) ${PAASelectors.clickTarget}`;
+    const count = await page.locator(unexpandedSel).count().catch(() => 0);
+    if (count === 0) return false;
+    for (let i = 0; i < count; i++) {
+      try {
+        const btn = page.locator(unexpandedSel).first();
+        await btn.scrollIntoViewIfNeeded();
+        await btn.click({ force: true });
+        await page.waitForTimeout(300);
+      } catch {
+      }
+    }
+    return true;
+  }
   async extractVideos(page) {
     const vsels = VideoSelectors;
     return page.evaluate((sels) => {
@@ -2074,12 +2126,58 @@ var PAAExtractor = class {
     }
     return { ...entityIds, entities: records, cids: [...cidSet] };
   }
-  async extractAISurfaces(page) {
-    return page.evaluate(extractAISurfacesFromDocument, {
+  async extractAISurfaces(page, options) {
+    const surfaces = await page.evaluate(extractAISurfacesFromDocument, {
       aio: AIOverviewSelectors,
       aim: AIModeSelectors,
       expandWaitMs: 1500
     });
+    if (!surfaces.aiOverview.detected) {
+      this.completeness.aioShareCaptured = null;
+      return surfaces;
+    }
+    const shareUrl = this.aioShareUrlEarly ?? await this.captureAIOverviewShareUrl(page, options).catch(() => null);
+    this.completeness.aioShareCaptured = shareUrl !== null;
+    return {
+      ...surfaces,
+      aiOverview: { ...surfaces.aiOverview, shareUrl }
+    };
+  }
+  async captureAIOverviewShareUrl(page, options) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      if (options?.softDeadlineMs && Date.now() >= options.softDeadlineMs) return null;
+      const clicked = await page.evaluate(() => {
+        const heads = Array.from(document.querySelectorAll('h1,h2,h3,[role="heading"],div')).filter((h) => (h.textContent || "").trim() === "AI Overview");
+        for (const head of heads) {
+          let sec = head;
+          for (let up = 0; up < 8 && sec; up++) {
+            const btn = sec.querySelector('[aria-label="Share"]');
+            if (btn) {
+              btn.scrollIntoView({ block: "center" });
+              btn.click();
+              return true;
+            }
+            sec = sec.parentElement;
+          }
+        }
+        return false;
+      }).catch(() => false);
+      if (!clicked) return null;
+      await page.waitForTimeout(4500);
+      const url = await page.evaluate(() => {
+        const w = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
+        while (w.nextNode()) {
+          const m = (w.currentNode.textContent || "").match(/https:\/\/share\.google\/aimode\/\S+/);
+          if (m) return m[0];
+        }
+        return null;
+      }).catch(() => null);
+      await page.keyboard.press("Escape").catch(() => {
+      });
+      if (url) return url;
+      await page.waitForTimeout(600);
+    }
+    return null;
   }
   buildTree(flat, _seed) {
     const roots = [];
@@ -2132,6 +2230,8 @@ var PAAExtractor = class {
   }
   async extract(options, signal) {
     const startMs = Date.now();
+    this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
+    this.aioShareUrlEarly = null;
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -2170,6 +2270,7 @@ var PAAExtractor = class {
       this.throwIfAborted(signal);
       const page = this.driver.getPage();
       await this.throwIfCaptcha(page, "Google SERP");
+      this.aioShareUrlEarly = await this.captureAIOverviewShareUrl(page, options).catch(() => null);
       if (options.serpOnly) {
         const [organicResults2, localPack2, rawEntityIds2] = await Promise.all([
           this.extractOrganicResults(page),
@@ -2177,7 +2278,7 @@ var PAAExtractor = class {
           this.extractEntityIds(page)
         ]);
         const entityIds2 = this.mergeLocalPackIntoEntities(rawEntityIds2, localPack2);
-        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const aiSurfaces2 = await this.extractAISurfaces(page, options);
         let locationEvidence2 = options.debug ? inferSerpLocationEvidence(canonicalLocation, organicResults2, localPack2) : void 0;
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
@@ -2206,6 +2307,7 @@ var PAAExtractor = class {
           diagnostics: {
             completionStatus: "serp_only",
             problem: null,
+            completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
@@ -2249,7 +2351,7 @@ var PAAExtractor = class {
             locationEvidence2 = inferSerpLocationEvidence(canonicalLocation, noPaaOrganic, localPack);
           }
         }
-        const aiSurfaces2 = await this.extractAISurfaces(page);
+        const aiSurfaces2 = await this.extractAISurfaces(page, options);
         const stats2 = {
           seed: options.query,
           totalQuestions: 0,
@@ -2265,6 +2367,7 @@ var PAAExtractor = class {
           diagnostics: {
             completionStatus: "no_paa",
             problem: null,
+            completeness: { ...this.completeness },
             ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence2) } : {}
           },
           totalQuestions: 0,
@@ -2284,7 +2387,7 @@ var PAAExtractor = class {
       }
       const flat = await this.runBFS(page, options, signal);
       this.throwIfAborted(signal);
-      const aiSurfaces = await this.extractAISurfaces(page);
+      const aiSurfaces = await this.extractAISurfaces(page, options);
       const shortVidsParams = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", udm: ShortVideoSelectors.udm });
       if (uule) shortVidsParams.set("uule", uule);
       let shortVideos = [];
@@ -2331,6 +2434,7 @@ var PAAExtractor = class {
         diagnostics: {
           completionStatus: "paa_found",
           problem: null,
+          completeness: { ...this.completeness },
           ...diagnosticWarnings.length > 0 ? { warnings: diagnosticWarnings } : {},
           ...options.debug ? { debug: this.buildHarvestDebugSnapshot(options, canonicalLocation, uule, locationEvidence) } : {}
         },