fetch_util 0.3.0

data/lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module CommunityAndMarketplace
+        COOKIE_BUTTON_SELECTORS = 'button, [role="button"], a, input[type="button"], input[type="submit"]'
+        EBAY_COOKIE_ACCEPT_LABELS = [
+          "accept all",
+          "accept all cookies",
+          "accept cookies",
+          "allow all",
+          "allow cookies",
+          "agree to cookies",
+          "continue with cookies"
+        ].freeze
+        private_constant :COOKIE_BUTTON_SELECTORS, :EBAY_COOKIE_ACCEPT_LABELS
+
+        private
+
+        def reddit_url?(url)
+          host_matches?(url, "reddit.com")
+        end
+
+        def stabilize_reddit(page)
+          retry_until_timeout(capped_timeout(3.0), interval: 0.1) do
+            dismiss_reddit_cookie_dialog(page)
+            reddit_content_ready?(page)
+          end
+
+          settle_after_stabilization(0.25)
+          dismiss_reddit_cookie_dialog(page)
+        end
+
+        def ebay_search_url?(url)
+          uri = URI.parse(url)
+          host = uri.host.to_s.downcase
+          return false unless host == "ebay.com" || host.end_with?(".ebay.com")
+
+          uri.path.include?("/sch/") || uri.query.to_s.include?("_nkw=")
+        rescue URI::InvalidURIError
+          false
+        end
+
+        def stabilize_ebay_search(page)
+          accepted_cookies = false
+
+          retry_until_timeout(capped_timeout(6.0), interval: 0.15) do
+            accepted_cookies ||= click_visible_button_by_text(
+              page,
+              EBAY_COOKIE_ACCEPT_LABELS,
+              selectors: COOKIE_BUTTON_SELECTORS
+            )
+
+            state = safe_evaluate(page, <<~JS, default: { "itemCount" => 0, "challengeVisible" => false })
+              (() => {
+                const bodyText = document.body ? document.body.innerText : '';
+                return {
+                  itemCount: document.querySelectorAll('li.s-item a[href*="/itm/"], ul.srp-results a[href*="/itm/"]').length,
+                  challengeVisible: /checking your browser before you access ebay|your browser will redirect to your requested content shortly|pardon our interruption/i.test(bodyText)
+                };
+              })()
+            JS
+
+            state["itemCount"].to_i >= 4 || (state["challengeVisible"] ? 0.35 : false)
+          end
+
+          settle_after_stabilization(0.25) if accepted_cookies
+        end
+
+        def reddit_content_ready?(page)
+          page.evaluate(<<~JS)
+            !!document.querySelector('shreddit-post, faceplate-screen-reader-content, shreddit-comment, [data-testid="comment"]')
+          JS
+        rescue Ferrum::JavaScriptError
+          false
+        end
+
+        def dismiss_reddit_cookie_dialog(page)
+          removed = dismiss_overlay_dialog(
+            page,
+            close_selectors: [],
+            dialog_selectors: [
+              '[data-testid="onboarding-modal"]',
+              '[data-testid="gdpr-modal"]',
+              '[aria-modal="true"]',
+              '[role="dialog"]',
+              "shreddit-experience-tree"
+            ],
+            dialog_pattern: "before you continue to reddit|let us know your cookie preferences"
+          )
+          return true if removed
+
+          safe_evaluate(page, <<~JS)
+            (() => {
+              #{js_dom_helpers}
+              let removed = false;
+              document.querySelectorAll('section, div, aside, form, footer, shreddit-experience-tree').forEach((node) => {
+                const text = (node.innerText || node.textContent || '').replace(/\s+/g, ' ').trim();
+                if (/before you continue to reddit|let us know your cookie preferences/i.test(text) && text.length < 2000) {
+                  node.remove();
+                  removed = true;
+                }
+              });
+
+              if (removed) {
+                restoreScroll();
+              }
+
+              return removed;
+            })()
+          JS
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization/social_platforms.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      module SocialPlatforms
+        COOKIE_BUTTON_SELECTORS = 'button, [role="button"], a, input[type="button"], input[type="submit"]'
+        INSTAGRAM_COOKIE_ACCEPT_LABELS = ["accept", "accept all", "accept all cookies"].freeze
+        INSTAGRAM_COOKIE_FALLBACK_LABELS = ["allow all cookies", "allow all", "allow cookies"].freeze
+        FACEBOOK_COOKIE_DECLINE_LABELS = [
+          "decline optional cookies",
+          "optionale cookies ablehnen",
+          "refuser les cookies optionnels",
+          "rechazar cookies opcionales",
+          "rifiuta i cookie opzionali"
+        ].freeze
+        FACEBOOK_COOKIE_ACCEPT_LABELS = [
+          "allow all cookies",
+          "alle cookies erlauben",
+          "autoriser tous les cookies",
+          "permitir todas las cookies",
+          "consenti tutti i cookie"
+        ].freeze
+        private_constant :COOKIE_BUTTON_SELECTORS, :INSTAGRAM_COOKIE_ACCEPT_LABELS, :INSTAGRAM_COOKIE_FALLBACK_LABELS,
+                         :FACEBOOK_COOKIE_DECLINE_LABELS, :FACEBOOK_COOKIE_ACCEPT_LABELS
+
+        private
+
+        def instagram_url?(url)
+          host_matches?(url, "instagram.com")
+        end
+
+        def stabilize_instagram(page)
+          wait_for_idle_or_content(page) if @wait_for_idle
+          accept_instagram_cookie_dialog(page) || accept_cookie_consent(page)
+          social_login_phase_pause
+          accept_instagram_cookie_dialog(page) || accept_cookie_consent(page)
+          retry_until_timeout(capped_timeout(5.0)) { dismiss_instagram_login_modal(page) }
+          social_login_phase_pause
+          dismiss_instagram_login_modal(page)
+        end
+
+        def accept_instagram_cookie_dialog(page)
+          click_visible_button_by_text(
+            page,
+            INSTAGRAM_COOKIE_ACCEPT_LABELS,
+            INSTAGRAM_COOKIE_FALLBACK_LABELS,
+            selectors: COOKIE_BUTTON_SELECTORS
+          )
+        end
+
+        def dismiss_instagram_login_modal(page)
+          dismiss_overlay_dialog(
+            page,
+            close_selectors: [
+              '[role="dialog"] button',
+              '[role="dialog"] [role="button"]',
+              '[role="dialog"] button[aria-label]',
+              '[role="dialog"] button[title]',
+              '[role="dialog"] button svg',
+              '[role="presentation"] button',
+              '[role="presentation"] [role="button"]',
+              '[role="presentation"] button[aria-label]',
+              '[role="presentation"] button[title]',
+              '[role="presentation"] button svg',
+              '[aria-modal="true"] button',
+              '[aria-modal="true"] [role="button"]',
+              '[aria-modal="true"] button[aria-label]',
+              '[aria-modal="true"] button[title]',
+              '[aria-modal="true"] button svg',
+              'div[style*="position: fixed"] button',
+              'div[style*="position:fixed"] button',
+              'div[style*="position: fixed"] [role="button"]',
+              'div[style*="position:fixed"] [role="button"]',
+              'div[style*="position: fixed"] svg',
+              'div[style*="position:fixed"] svg'
+            ],
+            dialog_selectors: ['[role="dialog"]', '[role="presentation"]', '[aria-modal="true"]'],
+            overlay_selectors: ['div[style*="position: fixed"]', 'div[style*="position:fixed"]'],
+            dialog_pattern: "log in|sign up|create (?:new )?account|don.?t have an account",
+            close_label_pattern: "^(?:close|dismiss|x|×)?$",
+            allow_empty_close_label: true
+          )
+        end
+
+        def facebook_url?(url)
+          host_matches?(url, "facebook.com")
+        end
+
+        def stabilize_facebook(page)
+          wait_for_idle_or_content(page) if @wait_for_idle
+          social_login_phase_pause
+          dismiss_facebook_cookie_dialog(page)
+          social_login_phase_pause
+          retry_until_timeout(capped_timeout(5.0)) { dismiss_facebook_login_dialog(page) }
+          social_login_phase_pause
+        end
+
+        def dismiss_facebook_cookie_dialog(page)
+          click_visible_button_by_text(
+            page,
+            FACEBOOK_COOKIE_DECLINE_LABELS,
+            FACEBOOK_COOKIE_ACCEPT_LABELS
+          )
+        end
+
+        def dismiss_facebook_login_dialog(page)
+          dismiss_overlay_dialog(
+            page,
+            close_selectors: ['[aria-label="Close"]', '[aria-label="close"]'],
+            dialog_selectors: ['[role="dialog"]', '[aria-modal="true"]'],
+            dialog_pattern: "log in|sign up|create (?:new )?account|see more from"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/site_stabilization.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module SiteStabilization
+      autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
+      autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
+
+      include CommunityAndMarketplace
+      include SocialPlatforms
+    end
+  end
+end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module Stabilization
+      module PageFlow
+        private
+
+        def stabilize_page(page, url)
+          return stabilize_reddit(page) if reddit_url?(url)
+          return stabilize_instagram(page) if instagram_url?(url)
+          return stabilize_facebook(page) if facebook_url?(url)
+          return stabilize_ebay_search(page) if ebay_search_url?(url)
+
+          reached_idle = !@wait_for_idle || wait_for_idle_or_content(page)
+          preserve_consent = preserve_consent_wall?(page, url)
+          accepted_cookies = preserve_consent ? false : accept_cookie_consent(page)
+          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+          sleep @wait if @wait.positive?
+          accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
+          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+
+          wait_for_spa_hydration(page) if @wait_for_idle && reached_idle
+          accepted_cookies = (!preserve_consent && accept_cookie_consent(page)) || accepted_cookies
+          accepted_cookies = (!preserve_consent && dismiss_privacy_preference_overlay(page)) || accepted_cookies
+
+          return unless accepted_cookies && @wait_for_idle && reached_idle
+
+          page.network.wait_for_idle(duration: @idle_duration, timeout: POST_CONSENT_IDLE_TIMEOUT)
+        end
+
+        def wait_for_idle_or_content(page)
+          content_seen_at = nil
+
+          retry_until_timeout(@timeout, interval: @idle_duration) do
+            now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            next true if page.network.idle?
+            if content_seen_at.nil? && page_has_content?(page)
+              content_seen_at = now
+            end
+
+            content_seen_at && (now - content_seen_at) >= @idle_duration
+          end
+        rescue Ferrum::Error
+          false
+        end
+
+        def page_has_content?(page)
+          page.evaluate(<<~JS)
+            (() => {
+              const body = document.body;
+              if (!body) return false;
+              const text = body.innerText || '';
+              return text.length > #{CONTENT_READY_MIN_LENGTH};
+            })()
+          JS
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
+
+        def preserve_consent_wall?(page, url)
+          host = FetchUtil.strip_www_host(url)
+          return false unless host == "youtube.com" || host.end_with?(".youtube.com") || host.match?(/\Agoogle\.[a-z.]+\z/)
+
+          state = page.evaluate(<<~JS)
+            (() => ({
+              title: document.title || '',
+              text: document.body ? document.body.innerText.replace(/\s+/g, ' ').trim().slice(0, 1200) : ''
+            }))()
+          JS
+
+          combined = [state["title"], state["text"]].join(" ").downcase
+          /before you continue to (google|youtube)|we use cookies and data|accept all|reject all|more options/.match?(combined)
+        rescue URI::InvalidURIError, Ferrum::JavaScriptError, Ferrum::TimeoutError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/stabilization/spa_hydration.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module Stabilization
+      # rubocop:disable Metrics/ModuleLength
+      module SpaHydration
+        private
+
+        def wait_for_spa_hydration(page)
+          framework = detect_spa_framework(page)
+          return unless framework
+
+          retry_until_timeout(SPA_HYDRATION_TIMEOUT, interval: SPA_HYDRATION_POLL) do
+            spa_hydration_complete?(page, framework)
+          end
+
+          sleep SPA_HYDRATION_POLL
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+        end
+
+        def detect_spa_framework(page)
+          result = page.evaluate(spa_framework_detection_script)
+
+          result.is_a?(String) ? result.to_sym : nil
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          nil
+        end
+
+        def spa_hydration_complete?(page, framework)
+          page.evaluate(spa_hydration_completion_script(framework))
+        rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
+          true
+        end
+
+        def spa_framework_detection_script
+          <<~JS
+            (() => {
+              if (window.__NEXT_DATA__ || (window.next && window.next.version)) return 'next';
+              if (window.__NUXT__ || window.$nuxt) return 'nuxt';
+              if (window.__remixContext) return 'remix';
+              if (document.getElementById('__docusaurus')) return 'docusaurus';
+              if (document.getElementById('___gatsby')) return 'gatsby';
+
+              const vueMount = document.querySelector('#app, #root, #__nuxt');
+              if ((vueMount && vueMount.__vue_app__) || window.__VUE__) return 'vue';
+              if (vueMount && vueMount.__vue__) return 'vue';
+              if (document.querySelector('[ng-version]')) return 'angular';
+              if (window.__svelte || document.getElementById('svelte-announcer') ||
+                  document.querySelector('[data-sveltekit-preload-data]') ||
+                  document.querySelector('[class*="svelte-"]')) return 'svelte';
+              if (window.Ember) return 'ember';
+              if (document.querySelector('[q\\:container]')) return 'qwik';
+              if (document.querySelector('meta[name="generator"][content*="Mintlify" i]') || document.querySelector('#content-area')) {
+                const gen = (document.querySelector('meta[name="generator"]') || {}).content || '';
+                if (/mintlify/i.test(gen) || document.querySelector('[class*="mintlify"]')) return 'mintlify';
+              }
+              {
+                const gen = (document.querySelector('meta[name="generator"]') || {}).content || '';
+                if (/gitbook/i.test(gen) || /\.gitbook\.io$/.test(location.hostname)) return 'gitbook';
+              }
+              if (document.querySelector('.scalar-api-reference, .scalar-app, [data-scalar]')) return 'scalar';
+              if (document.querySelector('redoc, rapi-doc, .redoc-wrap')) return 'redoc';
+              if (document.querySelector('meta[name="readme-deploy"]') || document.querySelector('.rm-Article, .rm-LandingPage, .rm-ReferenceMain')) return 'readme';
+
+              const reactCandidates = document.querySelectorAll('#root, #app, #__next, [data-reactroot], body > div');
+              for (const el of reactCandidates) {
+                if (el._reactRootContainer || Object.keys(el).some(k => k.startsWith('__reactContainer$'))) return 'react';
+              }
+
+              const mounts = document.querySelectorAll('#root, #app, [data-reactroot]');
+              if (mounts.length > 0) {
+                const bodyText = (document.body.innerText || '').trim();
+                if (bodyText.length < 500) return 'generic_spa';
+              }
+
+              return null;
+            })()
+          JS
+        end
+
+        def spa_hydration_completion_script(framework)
+          <<~JS
+            (() => {
+              const mainContent = document.querySelector('main, [role="main"], article, .content, .docs-content, #content');
+              const mainText = (mainContent ? mainContent.innerText : document.body.innerText || '').trim();
+
+              if (mainText.length > 500) return true;
+
+              switch ('#{framework}') {
+                case 'next': {
+                  const mount = document.getElementById('__next');
+                  if (mount && Object.keys(mount).some(k => k.startsWith('__reactFiber$'))) return true;
+                  if (window.next && window.next.version) {
+                    const bodyKids = document.body.children;
+                    for (let i = 0; i < bodyKids.length; i++) {
+                      if (Object.keys(bodyKids[i]).some(k => k.startsWith('__reactFiber$'))) return true;
+                    }
+                  }
+                  return false;
+                }
+                case 'nuxt': {
+                  const mount = document.getElementById('__nuxt');
+                  return !!(mount && mount.__vue_app__ && mount.__vue_app__._instance);
+                }
+                case 'vue': {
+                  const mount = document.querySelector('#app, #root');
+                  return !!(mount && (mount.__vue_app__ ? mount.__vue_app__._instance : mount.__vue__));
+                }
+                case 'react': {
+                  const candidates = document.querySelectorAll('#root, #app, [data-reactroot], body > div');
+                  for (const el of candidates) {
+                    if (Object.keys(el).some(k => k.startsWith('__reactFiber$'))) return true;
+                  }
+                  return false;
+                }
+                case 'angular': {
+                  const hasNgh = document.querySelectorAll('[ngh]').length > 0;
+                  if (hasNgh) return false;
+                  return !!document.querySelector('[ng-version]');
+                }
+                case 'svelte': {
+                  return !!document.getElementById('svelte-announcer') ||
+                         !!document.querySelector('[class*="svelte-"]');
+                }
+                case 'remix': {
+                  const candidates = document.querySelectorAll('#root, #app, body > div');
+                  for (const el of candidates) {
+                    if (Object.keys(el).some(k => k.startsWith('__reactFiber$'))) return true;
+                  }
+                  return !!window.__remixContext;
+                }
+                case 'ember': {
+                  return !!window.Ember && !!document.querySelector('.ember-view');
+                }
+                case 'qwik': {
+                  return !!document.querySelector('[q\\:container]');
+                }
+                case 'mintlify': {
+                  const contentArea = document.querySelector('#content-area, [id="content-area"]');
+                  if (contentArea && (contentArea.innerText || '').trim().length > 100) return true;
+                  return false;
+                }
+                case 'gitbook': {
+                  const gbMain = document.querySelector('main');
+                  if (gbMain && (gbMain.innerText || '').trim().length > 100) return true;
+                  return false;
+                }
+                case 'scalar': {
+                  const scalarRef = document.querySelector('.scalar-api-reference, .scalar-app');
+                  if (scalarRef && (scalarRef.innerText || '').trim().length > 500) return true;
+                  return false;
+                }
+                case 'redoc': {
+                  const apiContent = document.querySelector('.api-content, .redoc-wrap [role="main"], .redoc-wrap');
+                  if (apiContent && (apiContent.innerText || '').trim().length > 500) return true;
+                  return false;
+                }
+                case 'readme': {
+                  const rmArticle = document.querySelector('.rm-Article, .rm-LandingPage, article .markdown-body');
+                  if (rmArticle && (rmArticle.innerText || '').trim().length > 100) return true;
+                  return false;
+                }
+                case 'gatsby':
+                case 'docusaurus': {
+                  const mountId = '#{framework}' === 'gatsby' ? '___gatsby' : '__docusaurus';
+                  const mount = document.getElementById(mountId);
+                  return !!(mount && Object.keys(mount).some(k => k.startsWith('__reactFiber$')));
+                }
+                case 'generic_spa': {
+                  return mainText.length > 200;
+                }
+                default:
+                  return true;
+              }
+            })()
+          JS
+        end
+      end
+      # rubocop:enable Metrics/ModuleLength
+    end
+  end
+end

data/lib/fetch_util/browser/stabilization.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module Stabilization
+      autoload :PageFlow, "fetch_util/browser/stabilization/page_flow"
+      autoload :SpaHydration, "fetch_util/browser/stabilization/spa_hydration"
+
+      include PageFlow
+      include SpaHydration
+    end
+  end
+end

data/lib/fetch_util/browser.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "ferrum"
+require "json"
+require "uri"
+
+module FetchUtil
+  class Browser
+    autoload :InteractionHelpers, "fetch_util/browser/interaction_helpers"
+    autoload :Navigation, "fetch_util/browser/navigation"
+    autoload :SiteStabilization, "fetch_util/browser/site_stabilization"
+    autoload :Stabilization, "fetch_util/browser/stabilization"
+
+    include InteractionHelpers
+    include Navigation
+    include SiteStabilization
+    include Stabilization
+
+    # Prefer full Chromium over `headless_shell`: the full browser stays closer
+    # to standard Chromium behavior and exposes APIs that some sites expect,
+    # which improves extraction consistency. `headless_shell` diverges more
+    # often and can change page behavior in ways that degrade extraction.
+    BROWSER_CANDIDATES = [
+      "/usr/bin/chromium-browser",
+      "/usr/bin/chromium",
+      "/usr/bin/google-chrome",
+      "/usr/lib64/chromium-browser/headless_shell"
+    ].freeze
+
+    DEFAULT_VIEWPORT = { width: 1366, height: 900 }.freeze
+    DEFAULT_USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " \
+                         "(KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+    DEFAULT_ACCEPT_LANGUAGE = "en-US,en;q=0.9"
+    SOCIAL_LOGIN_PHASE_WAIT = 0.3
+    POST_CONSENT_IDLE_TIMEOUT = 3.0
+    CONTENT_READY_MIN_LENGTH = 200
+    SPA_HYDRATION_TIMEOUT = 2.0
+    SPA_HYDRATION_POLL = 0.15
+    NAVIGATION_MAX_RETRIES = 2
+
+    def initialize(timeout: 20, wait: 0.75, wait_for_idle: true, idle_duration: 0.35,
+                   viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT,
+                   accept_language: DEFAULT_ACCEPT_LANGUAGE, browser_path: nil,
+                   browser_options: nil)
+      @timeout = timeout.to_f
+      @wait = wait.to_f
+      @wait_for_idle = wait_for_idle
+      @idle_duration = idle_duration.to_f
+      @viewport = DEFAULT_VIEWPORT.merge(symbolize_hash(viewport || {}))
+      @user_agent = user_agent
+      @accept_language = accept_language
+      @browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
+      @full_browser = @browser_path && !@browser_path.include?("headless_shell")
+      default_opts = { "no-sandbox": nil }
+      # Use newer headless mode with the full browser binary for closer runtime
+      # parity with standard Chromium. Also disable Ferrum's
+      # `enable-automation` flag to reduce tool-specific browser-state
+      # differences during extraction.
+      if @full_browser
+        default_opts["headless"] = "new"
+        default_opts["enable-automation"] = false # override Ferrum default
+      end
+      @browser_options = default_opts.merge(browser_options || {})
+      @ferrum = nil
+      @mutex = Mutex.new
+    end
+
+    # Navigate to +url+ in a fresh browser tab, stabilize the page, then yield
+    # the +Ferrum::Page+ to the caller. The page is closed after the block
+    # returns (or on error), but the underlying Chromium process is kept alive
+    # for reuse by subsequent calls.
+    def with_page(url)
+      raise BrowserError, "No Chromium browser found. Set BROWSER_PATH or install Chromium." unless @browser_path
+
+      ferrum = ensure_browser
+      page = ferrum.create_page
+      page.headers.set(default_headers)
+      page.bypass_csp
+      retries = 0
+      begin
+        page.go_to(url)
+      rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError
+        unless page_loaded_enough?(page)
+          raise if retries >= NAVIGATION_MAX_RETRIES
+
+          retries += 1
+          retry
+        end
+      end
+      stabilize_page(page, url)
+      yield page
+    rescue Ferrum::Error => e
+      raise BrowserError, e.message
+    ensure
+      page&.close
+    end
+
+    # Shut down the underlying Chromium process. Safe to call multiple times or
+    # when no browser has been started yet. After +quit+, a subsequent
+    # +with_page+ call will transparently launch a new process.
+    def quit
+      @mutex.synchronize do
+        @ferrum&.quit
+        @ferrum = nil
+      end
+    end
+
+    private
+
+    # Lazily start the shared Chromium process on first use. The
+    # +evaluate_on_new_document+ call registers the navigator patch once;
+    # Ferrum automatically applies it to every new page/context created
+    # afterwards.
+    def ensure_browser
+      @mutex.synchronize do
+        return @ferrum if @ferrum
+
+        @ferrum = Ferrum::Browser.new(
+          headless: true,
+          browser_path: @browser_path,
+          timeout: @timeout,
+          window_size: [@viewport.fetch(:width), @viewport.fetch(:height)],
+          browser_options: @browser_options
+        )
+        @ferrum.evaluate_on_new_document(navigator_patch)
+        @ferrum
+      end
+    end
+
+    def host_matches?(url, host)
+      normalized_host = FetchUtil.strip_www_host(url)
+      normalized_host == host || normalized_host.end_with?(".#{host}")
+    end
+  end
+end