browserx 1.0.0 → 2.1.0

data/lib/browser/bot/ip_ranges.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+module Browser
+  class Bot
+    # The IP ranges crawler owners publish, from bot_ip_ranges.yml (rewritten
+    # weekly by script/sync_ip_ranges.rb) or fetched again by `refresh`.
+    module IPRanges
+      GOOGLE_RANGES_URL = "https://developers.google.com/static/crawling/ipranges"
+      SOURCES = {
+        google: {
+          keywords: %w[
+            adsbot-google apis-google feedfetcher-google google-cloudvertexbot google-inspectiontool
+            google-read-aloud google-safety google-site-verification googleagent-urlcontext googlebot
+            googleother googleproducer mediapartners-google storebot-google
+          ],
+          # https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot
+          domains: %w[googlebot.com google.com googleusercontent.com],
+          urls: %w[common-crawlers.json special-crawlers.json user-triggered-fetchers.json
+            user-triggered-fetchers-google.json user-triggered-agents.json].map { |file| "#{GOOGLE_RANGES_URL}/#{file}" }
+        },
+        bing: {
+          keywords: %w[adidxbot bingbot bingpreview microsoftpreview msnbot],
+          domains: %w[search.msn.com],
+          urls: %w[https://www.bing.com/toolbox/bingbot.json]
+        },
+        openai: {
+          keywords: %w[chatgpt-user gptbot oai-searchbot],
+          domains: [],
+          urls: %w[https://openai.com/gptbot.json https://openai.com/searchbot.json
+            https://openai.com/chatgpt-user.json]
+        },
+        apple: {
+          keywords: %w[applebot],
+          domains: %w[applebot.apple.com],
+          urls: %w[https://search.developer.apple.com/applebot.json]
+        },
+        perplexity: {
+          keywords: %w[perplexity-user perplexitybot],
+          domains: [],
+          urls: %w[https://www.perplexity.ai/perplexitybot.json https://www.perplexity.ai/perplexity-user.json]
+        },
+        duckduckgo: {
+          keywords: %w[duckassistbot duckduckbot],
+          domains: [],
+          urls: %w[https://duckduckgo.com/duckduckbot.json https://duckduckgo.com/duckassistbot.json]
+        }
+      }.freeze
+      FETCH_TIMEOUT_SECONDS = 10
+      # Owners move their files: Google's old URLs redirect to the new ones.
+      MAX_REDIRECTS = 3
+      DNS_TIMEOUT_SECONDS = 2
+
+      # The owner whose crawler the lowercase user agent claims to be.
+      def self.owner(ua)
+        SOURCES.each_key.find do |owner|
+          SOURCES[owner][:keywords].any? { |keyword| ua.include?(keyword) }
+        end
+      end
+
+      def self.include?(owner, ip)
+        address = parse(ip)
+        return false unless address
+
+        starts, ends = ranges_by_owner.dig(owner, address.family)
+        return false unless starts
+
+        number = address.to_i
+        index = (starts.bsearch_index { |start| start > number } || starts.size) - 1
+        index >= 0 && number <= ends[index]
+      end
+
+      # Google's documented method: the reverse lookup names one of the owner's
+      # domains, and that name resolves back to the IP. Two DNS queries.
+      def self.verified_by_dns?(owner, ip)
+        address = parse(ip)
+        domains = SOURCES.dig(owner, :domains)
+        return false unless address && domains&.any?
+
+        require "resolv"
+        Resolv::DNS.open do |dns|
+          dns.timeouts = DNS_TIMEOUT_SECONDS
+          host = dns.getname(address.to_s).to_s.downcase
+          domains.any? { |domain| host == domain || host.end_with?(".#{domain}") } &&
+            dns.getaddresses(host).any? { |resolved| IPAddr.new(resolved.to_s) == address }
+        end
+      rescue Resolv::ResolvError, Resolv::ResolvTimeout, SystemCallError
+        false
+      end
+
+      # Prefixes ("66.249.64.0/27"), keyed by owner.
+      def self.prefixes_by_owner
+        @prefixes_by_owner ||= Browser.load_data("bot_ip_ranges").transform_keys(&:to_sym)
+      end
+
+      def self.ranges_by_owner
+        @ranges_by_owner ||= build_ranges(prefixes_by_owner)
+      end
+
+      # For long-lived processes: fetches every owner's ranges again and swaps
+      # them in at once. Raises Browser::Error and keeps the current ranges when
+      # a source fails.
+      def self.refresh
+        load(fetch)
+      end
+
+      def self.load(prefixes_by_owner)
+        ranges_by_owner = build_ranges(prefixes_by_owner)
+        @prefixes_by_owner = prefixes_by_owner
+        @ranges_by_owner = ranges_by_owner
+      end
+
+      def self.fetch
+        SOURCES.to_h do |owner, source|
+          addresses = source[:urls].flat_map { |url| fetch_prefixes(url) }.map do |prefix|
+            IPAddr.new(prefix)
+          rescue ArgumentError
+            raise Error, "#{owner}: invalid prefix #{prefix.inspect}"
+          end
+          sorted = addresses.uniq.sort_by { |address| [address.family, address.to_i, address.prefix] }
+          [owner, sorted.map { |address| "#{address}/#{address.prefix}" }]
+        end
+      end
+
+      def self.fetch_prefixes(url)
+        require "json"
+        require "net/http"
+        uri = URI(url)
+        response = nil
+        (MAX_REDIRECTS + 1).times do
+          response = Net::HTTP.start(uri.host, uri.port, use_ssl: true,
+            open_timeout: FETCH_TIMEOUT_SECONDS, read_timeout: FETCH_TIMEOUT_SECONDS) do |http|
+            http.get(uri.request_uri, "User-Agent" => "BrowserX/#{VERSION} (+https://github.com/next-gen-gems/browserx)")
+          end
+          break unless response.is_a?(Net::HTTPRedirection)
+
+          uri = uri.merge(response["Location"])
+        end
+        raise Error, "#{url}: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+        prefixes = JSON.parse(response.body).fetch("prefixes").map do |prefix|
+          prefix["ipv4Prefix"] || prefix["ipv6Prefix"]
+        end
+        raise Error, "#{url}: no prefixes" if prefixes.empty?
+
+        prefixes
+      rescue JSON::ParserError, KeyError, Timeout::Error, IOError, SystemCallError, OpenSSL::SSL::SSLError => exception
+        raise Error, "#{url}: #{exception.message}"
+      end
+
+      # Per owner and address family, the merged ranges as two sorted arrays of
+      # integers, first and last addresses, for a binary search.
+      def self.build_ranges(prefixes_by_owner)
+        prefixes_by_owner.to_h do |owner, prefixes|
+          ranges_by_family = prefixes.map { |prefix| IPAddr.new(prefix) }.group_by(&:family)
+          [owner.to_sym, ranges_by_family.transform_values { |addresses| merge(addresses) }]
+        end
+      end
+
+      def self.merge(addresses)
+        ranges = addresses.map do |address|
+          range = address.to_range
+          [range.begin.to_i, range.end.to_i]
+        end
+        ranges.sort!
+        starts = []
+        ends = []
+        ranges.each do |first, last|
+          if ends.any? && first <= ends.last + 1
+            ends[-1] = last if last > ends.last
+          else
+            starts << first
+            ends << last
+          end
+        end
+        [starts, ends]
+      end
+
+      # A forged or malformed X-Forwarded-For must not raise in a request.
+      def self.parse(ip)
+        ip = ip.to_s
+        return if ip.include?("/")
+
+        address = IPAddr.new(ip)
+        address.ipv4_mapped? ? address.native : address
+      rescue ArgumentError
+        nil
+      end
+
+      private_class_method :fetch_prefixes, :build_ranges, :merge, :parse
+    end
+  end
+end

data/lib/browser/bot/known_bots_matcher.rb

@@ -4,7 +4,7 @@ module Browser
   class Bot
     class KnownBotsMatcher
       def self.call(ua, _browser)
-        ua.match?(Browser::Bot.keywords_pattern(Browser::Bot.bots))
+        !Browser::Bot.known_bot_rank(ua, lowest: false).nil?
       end
     end
   end

data/lib/browser/bot.rb

@@ -3,6 +3,8 @@
 module Browser
   class Bot
     GENERIC_NAME = "Generic Bot"
+    AI_CATEGORIES = %i[ai_agent ai_assistant ai_crawler ai_search].freeze
+    USER_TRIGGERED_CATEGORIES = %i[ai_agent ai_assistant].freeze
 
     def self.matchers
       @matchers ||= default_matchers
@@ -12,20 +14,33 @@ module Browser
       DEFAULT_MATCHERS.dup
     end
 
+    # browser's API. The gem ships its lists as JSON only: load_yaml("bots.yml")
+    # reads data/bots.json.
     def self.load_yaml(path)
+      json_path = Browser.root.join("data", path.to_s.sub(/\.yml\z/, ".json"))
+      return JSON.load_file(json_path) if json_path.file?
+
+      require "yaml"
       YAML.load_file(Browser.root.join(path))
     end
 
     def self.bots
-      @bots ||= load_yaml("bots.yml")
+      @bots ||= Browser.load_data("bots")
     end
 
     def self.bot_exceptions
-      @bot_exceptions ||= load_yaml("bot_exceptions.yml")
+      @bot_exceptions ||= Browser.load_data("bot_exceptions")
     end
 
     def self.search_engines
-      @search_engines ||= load_yaml("search_engines.yml")
+      @search_engines ||= Browser.load_data("search_engines")
+    end
+
+    def self.category_by_keyword
+      @category_by_keyword ||=
+        Browser.load_data("bot_categories").each_with_object({}) do |(category, keywords), category_by_keyword|
+          keywords.each { |keyword| category_by_keyword[keyword] = category.to_sym }
+        end
     end
 
     # One regexp per keyword list (an Array, or a Hash keyed by keyword),
@@ -41,6 +56,43 @@ module Browser
       pattern
     end
 
+    # bots.yml keywords nested by byte, each end holding the keyword's rank.
+    # Walked from each byte of a user agent, it follows the few branches that
+    # byte opens, where a regexp of a thousand keywords tries most of them.
+    def self.known_bots_trie
+      list, size, trie = @known_bots_trie
+      return trie if list.equal?(bots) && size == bots.size
+
+      trie = {keywords: bots.keys}
+      trie[:keywords].each_with_index do |keyword, rank|
+        leaf = keyword.to_s.each_byte.reduce(trie) { |node, byte| node[byte] ||= {} }
+        leaf[:rank] ||= rank
+      end
+      @known_bots_trie = [bots, bots.size, trie]
+      trie
+    end
+
+    # The rank of a bots.yml keyword the user agent holds: the lowest, or the
+    # first one found when `lowest: false`.
+    def self.known_bot_rank(ua, lowest: true)
+      trie = known_bots_trie
+      found = nil
+      ua.bytesize.times do |start|
+        node = trie
+        position = start
+        while (node = node[ua.getbyte(position)])
+          rank = node[:rank]
+          if rank
+            return rank unless lowest
+
+            found = rank if found.nil? || rank < found
+          end
+          position += 1
+        end
+      end
+      found
+    end
+
     # A Regexp.union of hundreds of words tries each of them at every
     # position; nested by common prefix, the regexp follows a single branch
     # per character.
@@ -76,8 +128,14 @@ module Browser
       @ua = ua.downcase.strip
     end
 
+    # A bot is shared by the browsers of its user agent: each answer holds
+    # until a matcher list or the bot list changes.
     def bot?
-      !bot_exception? && detect_bot?
+      generation = Cache.generation
+      unless @bot&.first == generation
+        @bot = [generation, !bot_exception? && detect_bot?]
+      end
+      @bot.last
     end
 
     def why?
@@ -91,7 +149,51 @@ module Browser
     def name
       return unless bot?
 
-      self.class.bots.find { |key, _| ua.include?(key) }&.last || GENERIC_NAME
+      generation = Cache.generation
+      unless @name&.first == generation
+        @name = [generation, self.class.bots[known_bot_keyword] || GENERIC_NAME]
+      end
+      @name.last
+    end
+
+    # Test drivers have a category, but no bot? (bot_categories.yml).
+    def category
+      return if bot_exception?
+
+      keyword = bot? && known_bot_keyword
+      keyword ||= ua[self.class.keywords_pattern(self.class.category_by_keyword)]
+      self.class.category_by_keyword[keyword]
+    end
+
+    def ai?
+      AI_CATEGORIES.include?(category)
+    end
+
+    def user_triggered?
+      USER_TRIGGERED_CATEGORIES.include?(category)
+    end
+
+    # The owner publishing the IP ranges of the crawler this user agent claims
+    # to be (:google, :bing, :openai, :apple, :perplexity, :duckduckgo).
+    def owner
+      return unless bot?
+
+      @owner = IPRanges.owner(ua) unless defined?(@owner)
+      @owner
+    end
+
+    # Whether the IP belongs to the owner's published ranges. reverse_dns: true
+    # also asks the DNS for an IP outside them (Google, Bing and Apple): two
+    # network queries.
+    def verified?(ip, reverse_dns: false)
+      return false unless owner
+
+      IPRanges.include?(owner, ip) || (reverse_dns && IPRanges.verified_by_dns?(owner, ip))
+    end
+
+    private def known_bot_keyword
+      rank = self.class.known_bot_rank(ua)
+      self.class.known_bots_trie[:keywords][rank] if rank
     end
 
     private def bot_exception?

data/lib/browser/browser.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
-require "yaml"
+if Gem.loaded_specs.key?("browser")
+  raise LoadError, "browserx replaces the browser gem, they can't be loaded together: " \
+    "remove gem \"browser\" from the Gemfile (or whatever depends on it)"
+end
+
+require "json"
 require "pathname"
 
 require_relative "version"
@@ -42,11 +47,16 @@ require_relative "bot"
 require_relative "bot/empty_user_agent_matcher"
 require_relative "bot/keyword_matcher"
 require_relative "bot/known_bots_matcher"
+require_relative "bot/ip_ranges"
 
 require_relative "middleware"
 require_relative "platform"
 require_relative "device"
 require_relative "meta"
+require_relative "client_hints"
+require_relative "cloudflare"
+require_relative "detection"
+require_relative "cache"
 
 module Browser
   EMPTY_STRING = ""
@@ -61,13 +71,41 @@ module Browser
     @root ||= Pathname.new(File.expand_path("../..", __dir__))
   end
 
+  # data/bots.json for "bots", written from bots.yml by script/build_data.rb: edit the YAML.
+  def self.load_data(name)
+    JSON.load_file(root.join("data", "#{name}.json"))
+  end
+
+  def self.eager_load!
+    [self, Platform, Device, Bot].each(&:matchers)
+    Meta.rules
+    AcceptLanguage.languages
+    Device::Samsung.names
+    Bot.known_bots_trie
+    [Bot.bot_exceptions, Bot.search_engines, Bot.category_by_keyword].each { |keywords| Bot.keywords_pattern(keywords) }
+    Bot::IPRanges.ranges_by_owner
+    nil
+  end
+
   class << self
     attr_accessor :user_agent_size_limit, :accept_language_size_limit
+    attr_reader :cache_size
   end
 
   self.user_agent_size_limit = 2048
   self.accept_language_size_limit = 2048
 
+  def self.cache_size=(size)
+    unless size.is_a?(Integer) && size >= 0
+      raise ArgumentError, "cache_size must be an Integer >= 0, got #{size.inspect}"
+    end
+
+    @cache_size = size
+    Cache.clear
+  end
+
+  self.cache_size = 1000
+
   # Hold the list of browser matchers.
   # Order is important.
   def self.matchers
@@ -107,10 +145,56 @@ module Browser
     ]
   end
 
+  # A pattern that every user agent a built-in browser matches also matches:
+  # Browser.new skips, without building it, a browser whose pattern fails.
+  REQUIRED_PATTERN_BY_MATCHER = {
+    Nokia => /S40OviBrowser/,
+    UCBrowser => /UCBrowser/,
+    PhantomJS => /PhantomJS/,
+    BlackBerry => /BlackBerry|BB10/,
+    Opera => %r{Opera|OPR/},
+    Edge => %r{Edg|Trident/8},
+    InternetExplorer => %r{MSIE|Trident/},
+    Firefox => /Firefox|FxiOS/,
+    Otter => /Otter/,
+    Epiphany => /Epiphany/,
+    Facebook => /FBAV|FBAN/,
+    Instagram => /Instagram/,
+    Snapchat => /Snapchat/,
+    Weibo => /__weibo__/i,
+    MicroMessenger => /MicroMessenger/i,
+    QQ => %r{QQ/|QQBrowser}i,
+    Alipay => /AlipayClient/i,
+    Electron => /Electron/,
+    Yandex => /YaBrowser/,
+    Sputnik => /SputnikBrowser/,
+    DuckDuckGo => /DuckDuckGo/,
+    SamsungBrowser => /SamsungBrowser/,
+    HuaweiBrowser => /HuaweiBrowser/i,
+    MiuiBrowser => /MiuiBrowser/,
+    Maxthon => /Maxthon/i,
+    SougouBrowser => /(?i:SogouMobileBrowser)|\bSE\b/,
+    GoogleSearchApp => /GSA/
+  }.compare_by_identity.freeze
+
   def self.new(user_agent, **)
+    user_agent ||= EMPTY_STRING
+    browser_class = Cache.cached_detection(user_agent)&.browser_class
+    return browser_class.new(user_agent, **) if browser_class
+
+    browser = detect(user_agent, **)
+    Cache.detection(user_agent).browser_class = browser.class
+    browser
+  end
+
+  def self.detect(user_agent, **)
     matchers.each do |matcher_class|
-      matcher = matcher_class.new(user_agent || EMPTY_STRING, **)
+      required_pattern = REQUIRED_PATTERN_BY_MATCHER[matcher_class]
+      next if required_pattern && !required_pattern.match?(user_agent)
+
+      matcher = matcher_class.new(user_agent, **)
       return matcher if matcher.match?
     end
   end
+  private_class_method :detect
 end

data/lib/browser/cache.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Browser
+  # The detections of the last user agents and the parsed Accept-Language
+  # headers, shared by the threads of a process. Reads take no lock: under the
+  # GVL a Hash read is atomic, and a thread that misses detects again. Writes
+  # do, for the Rubies without a GVL.
+  module Cache
+    @mutex = Mutex.new
+    @detection_by_user_agent = {}
+    @languages_by_header = {}
+    @generation = 0
+
+    def self.cached_detection(ua)
+      current_detection_by_user_agent[ua]
+    end
+
+    def self.detection(ua)
+      return Detection.new(ua) if Browser.cache_size.zero?
+
+      cached_detection(ua) || @mutex.synchronize do
+        ua = -ua
+        store(@detection_by_user_agent, ua, Detection.new(ua))
+      end
+    end
+
+    # A copy each time: callers may change the array they get.
+    def self.accept_language(header)
+      return AcceptLanguage.parse(header) if Browser.cache_size.zero?
+
+      languages = @languages_by_header[header] || @mutex.synchronize do
+        store(@languages_by_header, -header,
+          AcceptLanguage.parse(header).freeze)
+      end
+      languages.dup
+    end
+
+    # Bumped when a matcher list, the bot list or its exceptions change: the
+    # matchers are public arrays that apps extend at runtime.
+    def self.generation
+      return @generation if lists_unchanged?
+
+      @mutex.synchronize do
+        @generation += 1
+        @detection_by_user_agent.clear
+        remember_lists
+      end
+      @generation
+    end
+
+    def self.clear
+      @mutex.synchronize do
+        @detection_by_user_agent.clear
+        @languages_by_header.clear
+      end
+    end
+
+    # The bot list holds hundreds of entries: compared by identity and size,
+    # like Bot.keywords_pattern, not entry by entry on every request.
+    def self.lists_unchanged?
+      @browser_matchers == Browser.matchers &&
+        @platform_matchers == Platform.matchers &&
+        @device_matchers == Device.matchers &&
+        @bot_matchers == Bot.matchers &&
+        @bot_exceptions == Bot.bot_exceptions &&
+        @bots.equal?(Bot.bots) && @bots_size == Bot.bots.size
+    end
+
+    def self.remember_lists
+      @browser_matchers = Browser.matchers.dup
+      @platform_matchers = Platform.matchers.dup
+      @device_matchers = Device.matchers.dup
+      @bot_matchers = Bot.matchers.dup
+      @bot_exceptions = Bot.bot_exceptions.dup
+      @bots = Bot.bots
+      @bots_size = Bot.bots.size
+    end
+
+    def self.current_detection_by_user_agent
+      generation
+      @detection_by_user_agent
+    end
+
+    def self.store(value_by_key, key, value)
+      value_by_key.clear if value_by_key.size >= Browser.cache_size
+      value_by_key[key] = value
+    end
+
+    private_class_method :lists_unchanged?, :remember_lists,
+      :current_detection_by_user_agent, :store
+  end
+end

data/lib/browser/client_hints.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Browser
+  # The User-Agent Client Hints of Chromium browsers, read from a Rack env.
+  # Safari and Firefox send none: every answer is then nil.
+  class ClientHints
+    # A browser's hints are a few dozen bytes: longer values are ignored.
+    HEADER_SIZE_LIMIT_BYTES = 512
+    PLATFORM_CLASS_BY_NAME = {
+      "Android" => Platform::Android,
+      "Chrome OS" => Platform::ChromeOS,
+      "Linux" => Platform::Linux,
+      "macOS" => Platform::Mac,
+      "Windows" => Platform::Windows
+    }.freeze
+    # https://learn.microsoft.com/en-us/microsoft-edge/web-platform/how-to-detect-win11
+    WINDOWS_11_PLATFORM_VERSION = 13
+    BRAND_PATTERN = /"([^"\\]*)"\s*;\s*v\s*=\s*"([^"\\]*)"/
+    # GREASE brands, sent so that servers don't depend on the list's order.
+    GREASE_BRAND_PATTERN = /\ANot.A.Brand\z/i
+    QUOTED_STRING_PATTERN = /\A\s*"([^"\\]+)"\s*\z/
+    MOBILE_BY_VALUE = {"?1" => true, "?0" => false}.freeze
+
+    # Brand versions keyed by brand ("Google Chrome" => "142"), from Sec-CH-UA.
+    attr_reader :version_by_brand
+    # Sec-CH-UA-Platform ("Windows", "macOS", "Android"…).
+    attr_reader :platform
+    # Sec-CH-UA-Platform-Version ("15.0.0"), sent only to a site asking for it
+    # with an Accept-CH: Sec-CH-UA-Platform-Version response header.
+    attr_reader :platform_version
+
+    def initialize(headers)
+      @version_by_brand = parse_brands(header(headers, "HTTP_SEC_CH_UA"))
+      @mobile = MOBILE_BY_VALUE[header(headers, "HTTP_SEC_CH_UA_MOBILE")&.strip]
+      @platform = quoted_string(header(headers, "HTTP_SEC_CH_UA_PLATFORM"))
+      @platform_version = quoted_string(header(headers, "HTTP_SEC_CH_UA_PLATFORM_VERSION"))
+      freeze
+    end
+
+    # Sec-CH-UA-Mobile: nil when the browser did not send it.
+    def mobile?
+      @mobile
+    end
+
+    def platform_class
+      PLATFORM_CLASS_BY_NAME[platform]
+    end
+
+    # Windows 11 says "Windows NT 10.0" in its user agent: only its platform
+    # version tells it from Windows 10.
+    def windows11?
+      platform == "Windows" && platform_version.to_i >= WINDOWS_11_PLATFORM_VERSION
+    end
+
+    private def header(headers, name)
+      value = headers[name]
+      value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES
+    end
+
+    private def parse_brands(value)
+      return EMPTY_HASH unless value
+
+      value.scan(BRAND_PATTERN).to_h.reject { |brand, _version| GREASE_BRAND_PATTERN.match?(brand) }.freeze
+    end
+
+    private def quoted_string(value)
+      value&.[](QUOTED_STRING_PATTERN, 1)
+    end
+
+    EMPTY_HASH = {}.freeze
+    NONE = new(EMPTY_HASH)
+  end
+end