fetch_util 0.5.2 → 0.6.0

data/lib/fetch_util/browser/site_stabilization.rb

@@ -3,13 +3,102 @@
 module FetchUtil
   class Browser
     module SiteStabilization
+      autoload :AzureDevopsPrProductStateScript,
+               "fetch_util/browser/site_stabilization/azure_devops_pr_product_state_script"
+      autoload :AzureDevopsPrCommitDetailsScript,
+               "fetch_util/browser/site_stabilization/azure_devops_pr_commit_details_script"
+      autoload :AzureDevopsPrRequestHelpersScript,
+               "fetch_util/browser/site_stabilization/azure_devops_pr_request_helpers_script"
+      autoload :AzureDevopsPrRequestStateScript,
+               "fetch_util/browser/site_stabilization/azure_devops_pr_request_state_script"
+      autoload :AzureDevopsPrState, "fetch_util/browser/site_stabilization/azure_devops_pr_state"
+      autoload :AzureDevopsPrStabilization,
+               "fetch_util/browser/site_stabilization/azure_devops_pr_stabilization"
+      autoload :BitbucketCloudThreadState,
+               "fetch_util/browser/site_stabilization/bitbucket_cloud_thread_state"
+      autoload :BitbucketCloudThreads, "fetch_util/browser/site_stabilization/bitbucket_cloud_threads"
+      autoload :BitbucketCloudPullResourceModules,
+               "fetch_util/browser/site_stabilization/bitbucket_cloud_pull_resource_modules"
       autoload :CommunityAndMarketplace, "fetch_util/browser/site_stabilization/community_and_marketplace"
+      autoload :FacebookStabilization, "fetch_util/browser/site_stabilization/facebook_stabilization"
+      autoload :GerritChangeProductStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_change_product_state_script"
+      autoload :GerritChangeRequestStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_change_request_state_script"
+      autoload :GerritChangeState, "fetch_util/browser/site_stabilization/gerrit_change_state"
+      autoload :GerritChangeStabilization,
+               "fetch_util/browser/site_stabilization/gerrit_change_stabilization"
+      autoload :GerritFileResourceProductStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_product_state_script"
+      autoload :GerritFileResourceRouteStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_route_state_script"
+      autoload :GerritFileResourceFetchStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_fetch_state_script"
+      autoload :GerritFileResourceValidationStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_validation_state_script"
+      autoload :GerritFileResourceRequestStateScript,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_request_state_script"
+      autoload :GerritFileResourceState,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_state"
+      autoload :GerritFileResourceStabilization,
+               "fetch_util/browser/site_stabilization/gerrit_file_resource_stabilization"
+      autoload :GithubPullResources, "fetch_util/browser/site_stabilization/github_pull_resources"
+      autoload :GithubPullResourceStabilization,
+               "fetch_util/browser/site_stabilization/github_pull_resource_stabilization"
+      autoload :GithubThreads, "fetch_util/browser/site_stabilization/github_threads"
+      autoload :GiteaFamilyThreadProductState,
+               "fetch_util/browser/site_stabilization/gitea_family_thread_product_state"
+      autoload :GiteaFamilyThreadTimelineState,
+               "fetch_util/browser/site_stabilization/gitea_family_thread_timeline_state"
+      autoload :GiteaFamilyStabilization,
+               "fetch_util/browser/site_stabilization/gitea_family_stabilization"
+      autoload :GiteaFamilyVisibilityScript,
+               "fetch_util/browser/site_stabilization/gitea_family_visibility_script"
+      autoload :GiteaFamilyThreads, "fetch_util/browser/site_stabilization/gitea_family_threads"
+      autoload :GiteaFamilyPullResourceProductState,
+               "fetch_util/browser/site_stabilization/gitea_family_pull_resource_product_state"
+      autoload :GiteaFamilyPullResourceState,
+               "fetch_util/browser/site_stabilization/gitea_family_pull_resource_state"
+      autoload :GiteaFamilyPullResources, "fetch_util/browser/site_stabilization/gitea_family_pull_resources"
       autoload :GitlabRepo, "fetch_util/browser/site_stabilization/gitlab_repo"
+      autoload :GitlabMergeRequestResourceVisibilityScript,
+               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_visibility_script"
+      autoload :GitlabMergeRequestResourceStateScript,
+               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state_script"
+      autoload :GitlabMergeRequestResourceState,
+               "fetch_util/browser/site_stabilization/gitlab_merge_request_resource_state"
+      autoload :GitlabMergeRequestResources, "fetch_util/browser/site_stabilization/gitlab_merge_request_resources"
+      autoload :GitlabStabilization, "fetch_util/browser/site_stabilization/gitlab_stabilization"
+      autoload :GitlabThreads, "fetch_util/browser/site_stabilization/gitlab_threads"
       autoload :SocialPlatforms, "fetch_util/browser/site_stabilization/social_platforms"
       autoload :TravelAndLodging, "fetch_util/browser/site_stabilization/travel_and_lodging"
 
+      include AzureDevopsPrState
+      include AzureDevopsPrStabilization
+      include BitbucketCloudThreadState
+      include BitbucketCloudThreads
+      include BitbucketCloudPullResourceModules
       include CommunityAndMarketplace
+      include FacebookStabilization
+      include GerritChangeState
+      include GerritChangeStabilization
+      include GerritFileResourceState
+      include GerritFileResourceStabilization
+      include GithubPullResources
+      include GithubPullResourceStabilization
+      include GithubThreads
+      include GiteaFamilyVisibilityScript
+      include GiteaFamilyThreadProductState
+      include GiteaFamilyThreadTimelineState
+      include GiteaFamilyStabilization
+      include GiteaFamilyThreads
+      include GiteaFamilyPullResourceProductState
+      include GiteaFamilyPullResourceState
+      include GiteaFamilyPullResources
       include GitlabRepo
+      include GitlabStabilization
+      include GitlabMergeRequestResources
+      include GitlabThreads
       include SocialPlatforms
       include TravelAndLodging
     end

data/lib/fetch_util/browser/stabilization/anubis.rb

@@ -8,18 +8,17 @@ module FetchUtil
 
         private
 
-        def wait_for_anubis_challenge(page)
-          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
+        def wait_for_anubis_challenge(page, deadline: stabilization_deadline)
           state = anubis_page_state(page)
-          if !valid_anubis_state?(state) && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+          if !valid_anubis_state?(state) && monotonic_now < deadline
             state = anubis_page_state(page)
           end
           return false unless valid_anubis_state?(state) && state["challenge"] == true
 
-          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          remaining = capped_timeout(@timeout, deadline: deadline)
           return false unless remaining.positive?
 
-          retry_until_timeout(remaining, interval: ANUBIS_POLL) do
+          retry_until_timeout(remaining, interval: ANUBIS_POLL, deadline: deadline) do
             state = anubis_page_state(page)
             next false unless valid_anubis_state?(state)
             next false unless state["challenge"] == false

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -10,6 +10,17 @@ module FetchUtil
           SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_instagram],
           SiteStabilization::SocialPlatforms::SOCIAL_PLATFORM_STABILIZATION_PROFILES[:stabilize_facebook],
           SiteStabilization::CommunityAndMarketplace::COMMUNITY_MARKETPLACE_STABILIZATION_PROFILES[:stabilize_ebay_search],
+          SiteStabilization::GithubThreads::GITHUB_THREAD_STABILIZATION_PROFILE,
+          SiteStabilization::GithubPullResources::GITHUB_PULL_RESOURCE_STABILIZATION_PROFILE,
+          SiteStabilization::GitlabThreads::GITLAB_THREAD_STABILIZATION_PROFILE,
+          SiteStabilization::GitlabMergeRequestResources::GITLAB_MERGE_REQUEST_RESOURCE_STABILIZATION_PROFILE,
+          SiteStabilization::GiteaFamilyPullResources::GITEA_FAMILY_PULL_RESOURCE_STABILIZATION_PROFILE,
+          SiteStabilization::GiteaFamilyThreads::GITEA_FAMILY_THREAD_STABILIZATION_PROFILE,
+          SiteStabilization::BitbucketCloudPullResources::BITBUCKET_CLOUD_PULL_RESOURCE_STABILIZATION_PROFILE,
+          SiteStabilization::BitbucketCloudThreads::BITBUCKET_CLOUD_THREAD_STABILIZATION_PROFILE,
+          SiteStabilization::AzureDevopsPrStabilization::AZURE_DEVOPS_PR_STABILIZATION_PROFILE,
+          SiteStabilization::GerritFileResourceStabilization::GERRIT_FILE_RESOURCE_STABILIZATION_PROFILE,
+          SiteStabilization::GerritChangeStabilization::GERRIT_CHANGE_STABILIZATION_PROFILE,
           SiteStabilization::TravelAndLodging::TRAVEL_LODGING_STABILIZATION_PROFILES[:stabilize_lodging_detail],
           { host: "t.me", path_query: ->(uri) { uri.path.match?(%r{\A/s/[^/]+/\d+/?\z}) },
             strategy: :wait_for_telegram_message, notes: "Wait for the requested public Telegram preview message.",
@@ -38,31 +49,48 @@ module FetchUtil
         private
 
         def stabilize_page(page, url)
-          wait_for_anubis_challenge(page)
+          deadline = stabilization_deadline
+          wait_for_anubis_challenge(page, deadline: deadline)
+          return false unless stabilization_time_remaining?(deadline)
 
           if (profile = matching_stabilization_profile(url, PAGE_FLOW_STABILIZATION_PROFILES))
-            return send(profile.fetch(:strategy), page)
+            handled = send(profile.fetch(:strategy), page, deadline: deadline)
+            return handled unless profile[:fallthrough] && !handled
+            return false unless stabilization_time_remaining?(deadline)
           end
 
-          reached_idle = !@wait_for_idle || wait_for_idle_or_content(page)
+          reached_idle = !@wait_for_idle || wait_for_idle_or_content(page, deadline: deadline)
+          return false unless stabilization_time_remaining?(deadline)
+
           preserve_consent = preserve_consent_wall?(page, url)
+          return false unless stabilization_time_remaining?(deadline)
+
           accepted_cookies = preserve_consent ? false : dismiss_cookie_overlays(page)
+          return false unless stabilization_time_remaining?(deadline)
+
           if @wait.positive? && (!@wait_for_idle || accepted_cookies)
-            sleep @wait
+            sleep_before_deadline(@wait, deadline: deadline)
+            return false unless stabilization_time_remaining?(deadline)
+
             accepted_cookies = dismiss_cookie_overlays(page) || accepted_cookies unless preserve_consent
+            return false unless stabilization_time_remaining?(deadline)
           end
 
-          wait_for_spa_hydration(page) if @wait_for_idle && reached_idle
+          wait_for_spa_hydration(page, deadline: deadline) if @wait_for_idle && reached_idle
+          return false unless stabilization_time_remaining?(deadline)
+
           if accepted_cookies
             accepted_cookies = dismiss_cookie_overlays(page) || accepted_cookies
+            return false unless stabilization_time_remaining?(deadline)
           end
           if (profile = matching_stabilization_profile(url, POST_GENERIC_STABILIZATION_PROFILES))
-            send(profile.fetch(:strategy), page, url)
+            send(profile.fetch(:strategy), page, url, deadline: deadline)
+            return false unless stabilization_time_remaining?(deadline)
           end
 
           return unless accepted_cookies && @wait_for_idle && reached_idle
 
-          wait_for_network_idle(page)
+          wait_for_network_idle(page, deadline: deadline)
         end
 
         def matching_stabilization_profile(url, profiles)
@@ -75,13 +103,18 @@ module FetchUtil
 
         def profile_match?(profile, uri, host) = stabilization_host_matches?(profile.fetch(:host), host) && profile.fetch(:path_query, ->(_) { true }).call(uri)
 
-        def stabilization_host_matches?(matcher, host) = Array(matcher).any? { |candidate| host == candidate || host.end_with?(".#{candidate}") }
+        def stabilization_host_matches?(matcher, host)
+          return true if matcher == true
 
-        def wait_for_idle_or_content(page)
+          Array(matcher).any? { |candidate| host == candidate || host.end_with?(".#{candidate}") }
+        end
+
+        def wait_for_idle_or_content(page, deadline: stabilization_deadline)
           content_seen_at = nil
 
-          retry_until_timeout(@timeout, interval: @idle_duration) do
-            now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          retry_until_timeout(capped_timeout(@timeout, deadline: deadline),
+                              interval: @idle_duration, deadline: deadline) do
+            now = monotonic_now
             next true if page.network.idle?
             if content_seen_at.nil? && page_has_content?(page)
               content_seen_at = now
@@ -93,11 +126,19 @@ module FetchUtil
           false
         end
 
-        def wait_for_network_idle(page)
-          page.network.wait_for_idle(duration: @idle_duration, timeout: POST_CONSENT_IDLE_TIMEOUT)
-          true
-        rescue Ferrum::TimeoutError
-          false
+        def wait_for_network_idle(page, deadline: stabilization_deadline)
+          idle_since = nil
+          timeout = capped_timeout(POST_CONSENT_IDLE_TIMEOUT, deadline: deadline)
+          retry_until_timeout(timeout, interval: @idle_duration, deadline: deadline) do
+            now = monotonic_now
+            if page.network.idle?
+              idle_since ||= now
+              now >= idle_since + @idle_duration
+            else
+              idle_since = nil
+              false
+            end
+          end
         rescue Ferrum::Error => e
           raise if retryable_pending_connections_error?(e)
 
@@ -135,8 +176,8 @@ module FetchUtil
           false
         end
 
-        def wait_for_agora_article(page, _url)
-          retry_until_timeout(7.0, interval: 0.25) do
+        def wait_for_agora_article(page, _url, deadline: stabilization_deadline)
+          retry_until_timeout(capped_timeout(7.0, deadline: deadline), interval: 0.25, deadline: deadline) do
             page.evaluate(<<~JS)
               (() => {
                 const selectors = ['.mrf-article-body', '.article_body', 'div.articleBody', '.art_content', '.article-inner', 'section.article', '[itemprop=articleBody]'];
@@ -150,8 +191,8 @@ module FetchUtil
           false
         end
 
-        def wait_for_france24_article(page, _url)
-          retry_until_timeout(15.0, interval: 0.25) do
+        def wait_for_france24_article(page, _url, deadline: stabilization_deadline)
+          retry_until_timeout(capped_timeout(15.0, deadline: deadline), interval: 0.25, deadline: deadline) do
             page.evaluate(<<~JS)
               (() => {
                 const body = document.querySelector('.t-content__body') || document.querySelector('.t-content--article');
@@ -165,18 +206,21 @@ module FetchUtil
           false
         end
 
-        def wait_for_onet_homepage(page)
-          wait_for_structural_readiness(page, "section[class*='Feed_']", "article[class*='Card_']")
+        def wait_for_onet_homepage(page, deadline: stabilization_deadline)
+          wait_for_structural_readiness(page, "section[class*='Feed_']", "article[class*='Card_']",
+                                        deadline: deadline)
         end
 
-        def wait_for_wp_homepage(page)
-          wait_for_structural_readiness(page, ".wp-section-grid", ".wp-teaser-tile, .wp-teaser-regular")
+        def wait_for_wp_homepage(page, deadline: stabilization_deadline)
+          wait_for_structural_readiness(page, ".wp-section-grid", ".wp-teaser-tile, .wp-teaser-regular",
+                                        deadline: deadline)
         end
 
-        def wait_for_telegram_message(page)
+        def wait_for_telegram_message(page, deadline: stabilization_deadline)
           target = URI.parse(page.current_url).path.delete_prefix("/s/")
 
-          ready = retry_until_timeout(capped_timeout(5.0), interval: 0.25) do
+          ready = retry_until_timeout(capped_timeout(5.0, deadline: deadline),
+                                      interval: 0.25, deadline: deadline) do
             page.evaluate(<<~JS)
               (() => {
                 const card = document.querySelector('.tgme_widget_message[data-post="#{target}"]');
@@ -185,7 +229,7 @@ module FetchUtil
               })()
             JS
           end
-          sleep PRE_EXTRACTION_SETTLE_WAIT if ready
+          sleep_before_deadline(PRE_EXTRACTION_SETTLE_WAIT, deadline: deadline) if ready
           ready
         rescue URI::InvalidURIError, Ferrum::JavaScriptError, Ferrum::TimeoutError
           false

data/lib/fetch_util/browser/stabilization/spa_hydration.rb

@@ -7,21 +7,23 @@ module FetchUtil
       module SpaHydration
         private
 
-        def wait_for_spa_hydration(page)
+        def wait_for_spa_hydration(page, deadline: stabilization_deadline)
           framework = detect_spa_framework(page)
           return unless framework
 
-          retry_until_timeout(SPA_HYDRATION_TIMEOUT, interval: SPA_HYDRATION_POLL) do
+          retry_until_timeout(capped_timeout(SPA_HYDRATION_TIMEOUT, deadline: deadline),
+                              interval: SPA_HYDRATION_POLL, deadline: deadline) do
             spa_hydration_complete?(page, framework)
           end
 
-          sleep SPA_HYDRATION_POLL
+          sleep_before_deadline(SPA_HYDRATION_POLL, deadline: deadline)
         rescue Ferrum::JavaScriptError, Ferrum::TimeoutError
         end
 
-        def wait_for_structural_readiness(page, region_selector, card_selector)
+        def wait_for_structural_readiness(page, region_selector, card_selector, deadline: stabilization_deadline)
           previous_count = nil
-          retry_until_timeout(SPA_HYDRATION_TIMEOUT, interval: SPA_HYDRATION_POLL) do
+          retry_until_timeout(capped_timeout(SPA_HYDRATION_TIMEOUT, deadline: deadline),
+                              interval: SPA_HYDRATION_POLL, deadline: deadline) do
             counts = page.evaluate(<<~JS)
               (() => ({
                 regions: document.querySelectorAll(#{region_selector.to_json}).length,

data/lib/fetch_util/browser.rb

@@ -45,18 +45,19 @@ module FetchUtil
                    viewport: DEFAULT_VIEWPORT, user_agent: DEFAULT_USER_AGENT,
                    accept_language: DEFAULT_ACCEPT_LANGUAGE, browser_path: nil,
                    browser_options: nil)
-      @timeout = timeout.to_f
-      @wait = wait.to_f
+      @timeout = positive_timeout(timeout)
+      @wait = nonnegative_duration(wait, "wait")
       @wait_for_idle = wait_for_idle
-      @idle_duration = idle_duration.to_f
+      @idle_duration = nonnegative_duration(idle_duration, "idle_duration")
       @viewport = DEFAULT_VIEWPORT.merge(symbolize_hash(viewport || {}))
-      @user_agent = user_agent
-      @accept_language = accept_language
+      @user_agent = user_agent.to_s.dup.freeze
+      @accept_language = accept_language.to_s.dup.freeze
       @default_headers = {
         "User-Agent" => @user_agent,
         "Accept-Language" => @accept_language
       }.freeze
-      @browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
+      selected_browser_path = browser_path || ENV["BROWSER_PATH"] || BROWSER_CANDIDATES.find { |path| File.executable?(path) }
+      @browser_path = selected_browser_path&.dup&.freeze
       @full_browser = @browser_path && !@browser_path.include?("headless_shell")
       default_opts = { "no-sandbox": nil }
       # Use newer headless mode with the full browser binary for closer runtime
@@ -67,9 +68,10 @@ module FetchUtil
         default_opts["headless"] = "new"
         default_opts["enable-automation"] = false # override Ferrum default
       end
-      @browser_options = default_opts.merge(browser_options || {})
+      @browser_options = immutable_browser_option(default_opts.merge(browser_options || {}))
       @navigator_patch = build_navigator_patch
       @ferrum = nil
+      @pending_shutdown = nil
       @mutex = Mutex.new
     end
 
@@ -82,12 +84,12 @@ module FetchUtil
 
       normalized_url = FetchUtil.normalize_url(url)
       page = load_page_with_retry(ensure_browser, normalized_url)
-      sleep PRE_EXTRACTION_SETTLE_WAIT if heavy_script_page?(page, normalized_url)
+      sleep PRE_EXTRACTION_SETTLE_WAIT if heavy_script_page?(page, loaded_page_url(page, normalized_url))
       yield page
     rescue Ferrum::Error => e
       raise BrowserError, e.message
     ensure
-      page&.close
+      close_page(page)
     end
 
     # Shut down the underlying Chromium process. Safe to call multiple times or
@@ -95,33 +97,92 @@ module FetchUtil
     # +with_page+ call will transparently launch a new process.
     def quit
       @mutex.synchronize do
-        @ferrum&.quit
+        browser = @ferrum || @pending_shutdown
         @ferrum = nil
+        @pending_shutdown = browser
+        shutdown_pending_browser
       end
     end
 
     private
 
+    def positive_timeout(value)
+      timeout = Float(value)
+      return timeout if timeout.positive? && timeout.finite?
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError
+      raise InputError, "timeout must be positive"
+    end
+
+    def nonnegative_duration(value, name)
+      duration = Float(value)
+      return duration if duration >= 0 && duration.finite?
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError
+      raise InputError, "#{name} must be nonnegative"
+    end
+
+    def immutable_browser_option(value, memo = {}.compare_by_identity)
+      return memo[value] if memo.key?(value)
+
+      case value
+      when String
+        memo[value] = value.dup.freeze
+      when Array
+        owned = []
+        memo[value] = owned
+        value.each { |item| owned << immutable_browser_option(item, memo) }
+        owned.freeze
+      when Hash
+        owned = {}
+        memo[value] = owned
+        value.each do |key, item|
+          owned[immutable_browser_option(key, memo)] = immutable_browser_option(item, memo)
+        end
+        owned.freeze
+      else
+        value
+      end
+    end
+
     # Lazily start the shared Chromium process on first use. The
     # +evaluate_on_new_document+ call registers the navigator patch once;
     # Ferrum automatically applies it to every new page/context created
     # afterwards.
     def ensure_browser
       @mutex.synchronize do
+        shutdown_pending_browser
         return @ferrum if @ferrum
 
-        @ferrum = Ferrum::Browser.new(
+        browser = Ferrum::Browser.new(
           headless: true,
           browser_path: @browser_path,
           timeout: @timeout,
           window_size: [@viewport.fetch(:width), @viewport.fetch(:height)],
           browser_options: @browser_options
         )
-        @ferrum.evaluate_on_new_document(navigator_patch)
-        @ferrum
+        browser.evaluate_on_new_document(navigator_patch)
+        @ferrum = browser
+      rescue Ferrum::Error
+        @pending_shutdown = browser
+        begin
+          shutdown_pending_browser
+        rescue Ferrum::Error
+          nil
+        end
+        raise
       end
     end
 
+    def shutdown_pending_browser
+      return unless @pending_shutdown
+
+      @pending_shutdown.quit
+      @pending_shutdown = nil
+    end
+
     def host_matches?(url, host)
       normalized_host = FetchUtil.strip_www_host(url)
       normalized_host == host || normalized_host.end_with?(".#{host}")
@@ -132,8 +193,7 @@ module FetchUtil
     end
 
     def retryable_navigation_error?(error)
-      error.is_a?(Ferrum::PendingConnectionsError) || error.is_a?(Ferrum::TimeoutError) ||
-        error.message.to_s.match?(/pending connections/i)
+      error.is_a?(Ferrum::TimeoutError) || retryable_pending_connections_error?(error)
     end
 
     def heavy_script_page?(page, url)
@@ -146,6 +206,17 @@ module FetchUtil
       false
     end
 
+    def close_page(page)
+      page&.close
+    rescue Ferrum::Error
+      nil
+    end
+
+    def loaded_page_url(page, fallback)
+      current_url = page.current_url
+      current_url.to_s.empty? ? fallback : current_url
+    end
+
     def load_page_with_retry(ferrum, url)
       retries = 0
 
@@ -158,10 +229,10 @@ module FetchUtil
         rescue Ferrum::TimeoutError
           raise unless page_loaded_enough?(page)
         end
-        stabilize_page(page, url)
+        stabilize_page(page, loaded_page_url(page, url))
         page
       rescue Ferrum::PendingConnectionsError, Ferrum::TimeoutError, Ferrum::Error => e
-        page&.close
+        close_page(page)
         raise unless retryable_navigation_error?(e)
         raise if retries >= NAVIGATION_MAX_RETRIES
 

data/lib/fetch_util/cli.rb

@@ -6,6 +6,31 @@ require "thor"
 
 module FetchUtil
   class CLI < Thor
+    class << self
+      def start(given_args = ARGV, config = {})
+        command = given_args.first
+        arguments = given_args.drop(1)
+        help_command = command_for_local_help(command, arguments)
+        return super(["help", help_command], config) if help_command
+
+        super
+      end
+
+      def exit_on_failure? = true
+
+      private
+
+      def command_for_local_help(command, arguments)
+        normalized_command = normalize_command_name(command)
+        return unless all_commands.key?(normalized_command)
+
+        help_requested = arguments.take_while { |argument| argument != "--" }.any? do |argument|
+          Thor::HELP_MAPPINGS.include?(argument)
+        end
+        normalized_command if help_requested
+      end
+    end
+
     DEFAULT_FETCH_FIELDS = %i[
       title
       byline
@@ -55,13 +80,15 @@ module FetchUtil
 
     desc "fetch URL [URL...]", "Fetch one or more URLs"
     def fetch(*urls)
-      raise ArgumentError, "at least one URL is required" if urls.empty?
+      raise Thor::Error, "at least one URL is required" if urls.empty?
 
-      results = if urls.length == 1
-                  [FetchUtil.fetch(urls.first, **fetch_options, request_log: request_log)]
-                else
-                  FetchUtil.fetch_many(urls, **fetch_options, request_log: request_log, concurrency: options[:concurrency])
-                end
+      results = translate_input_errors do
+        if urls.length == 1
+          [FetchUtil.fetch(urls.first, **fetch_options, request_log: request_log)]
+        else
+          FetchUtil.fetch_many(urls, **fetch_options, request_log: request_log, concurrency: options[:concurrency])
+        end
+      end
 
       if options[:format] == "markdown"
         puts results.map { |result| front_matter_document(result) }.join("\n\n")
@@ -76,15 +103,18 @@ module FetchUtil
     option :verbose_search, type: :boolean, default: false, desc: "Include per-result search provenance"
     def search(*terms)
       query = terms.join(" ").strip
-      raise ArgumentError, "query is required" if query.empty?
-
-      payload = Searcher.new(
-        request_log: request_log,
-        sources: options[:source],
-        limit: options[:limit],
-        verbose: options[:verbose_search],
-        timeout: options[:timeout]
-      ).search(query)
+      raise Thor::Error, "query is required" if query.empty?
+
+      searcher = translate_input_errors do
+        Searcher.new(
+          request_log: request_log,
+          sources: options[:source],
+          limit: options[:limit],
+          verbose: options[:verbose_search],
+          timeout: options[:timeout]
+        )
+      end
+      payload = translate_input_errors { searcher.search(query) }
 
       emit(payload)
     end
@@ -93,22 +123,38 @@ module FetchUtil
     option :sources, type: :string, default: "machine", desc: "Comma-separated source selectors, e.g. machine,-robotstxt or human,machine,-human"
     option :cache_path, type: :string, desc: "Structured regulatory cache directory"
     def regulatory(url)
-      raise ArgumentError, "url is required" if url.to_s.strip.empty?
+      raise Thor::Error, "url is required" if url.to_s.strip.empty?
 
       request_log.append("regulatory://#{url}?sources=#{options[:sources]}")
-      payload = FetchUtil.regulatory(
-        url,
-        cache_path: options[:cache_path],
-        sources: options[:sources],
-        timeout: options[:timeout]
-      )
+      payload = translate_input_errors do
+        FetchUtil.regulatory(
+          url,
+          cache_path: options[:cache_path],
+          sources: options[:sources],
+          timeout: options[:timeout]
+        )
+      end
 
       emit(payload)
     end
 
     no_commands do
       def request_log
-        @request_log ||= RequestLog.new(path: options[:log_path] || ENV.fetch("FETCH_UTIL_REQUEST_LOG", RequestLog::DEFAULT_PATH))
+        @request_log ||= options[:log_path] ? RequestLog.new(path: options[:log_path]) : RequestLog.new
+      end
+
+      def translate_input_errors
+        yield
+      rescue ParallelFetcher::ParallelFetchError => e
+        raise unless e.errors.all? { |error| input_error?(error) }
+
+        raise Thor::Error, e.message
+      rescue URI::InvalidURIError, InputError => e
+        raise Thor::Error, e.message
+      end
+
+      def input_error?(error)
+        error.is_a?(URI::InvalidURIError) || error.is_a?(InputError)
       end
 
       def fetch_options