agentry 3.0.0

data/lib/browser/bot.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    GENERIC_NAME = "Generic Bot"
+    AI_CATEGORIES = %i[ai_agent ai_assistant ai_crawler ai_search].freeze
+    USER_TRIGGERED_CATEGORIES = %i[ai_agent ai_assistant].freeze
+
+    def self.matchers
+      @matchers ||= default_matchers
+    end
+
+    def self.default_matchers
+      DEFAULT_MATCHERS.dup
+    end
+
+    # browser's API. The gem ships its lists as JSON only: load_yaml("bots.yml")
+    # reads data/bots.json.
+    def self.load_yaml(path)
+      json_path = Browser.root.join("data", path.to_s.sub(/\.yml\z/, ".json"))
+      return JSON.load_file(json_path) if json_path.file?
+
+      require "yaml"
+      YAML.load_file(Browser.root.join(path))
+    end
+
+    def self.bots
+      @bots ||= Browser.load_data("bots")
+    end
+
+    def self.bot_exceptions
+      @bot_exceptions ||= Browser.load_data("bot_exceptions")
+    end
+
+    def self.search_engines
+      @search_engines ||= Browser.load_data("search_engines")
+    end
+
+    def self.category_by_keyword
+      @category_by_keyword ||=
+        Browser.load_data("bot_categories").each_with_object({}) do |(category, keywords), category_by_keyword|
+          keywords.each { |keyword| category_by_keyword[keyword] = category.to_sym }
+        end
+    end
+
+    # One regexp per keyword list (an Array, or a Hash keyed by keyword),
+    # rebuilt when the list grows or is replaced.
+    def self.keywords_pattern(keywords)
+      @keywords_pattern_by_list ||= {}.compare_by_identity
+      size, pattern = @keywords_pattern_by_list[keywords]
+      return pattern if size == keywords.size
+
+      words = (keywords.is_a?(Hash) ? keywords.keys : keywords).map(&:to_s)
+      pattern = Regexp.new(trie_source(words))
+      @keywords_pattern_by_list[keywords] = [keywords.size, pattern]
+      pattern
+    end
+
+    # bots.yml keywords nested by byte, each end holding the keyword's rank.
+    # Walked from each byte of a user agent, it follows the few branches that
+    # byte opens, where a regexp of a thousand keywords tries most of them.
+    def self.known_bots_trie
+      list, size, trie = @known_bots_trie
+      return trie if list.equal?(bots) && size == bots.size
+
+      trie = {keywords: bots.keys}
+      trie[:keywords].each_with_index do |keyword, rank|
+        leaf = keyword.to_s.each_byte.reduce(trie) { |node, byte| node[byte] ||= {} }
+        leaf[:rank] ||= rank
+      end
+      @known_bots_trie = [bots, bots.size, trie]
+      trie
+    end
+
+    # The rank of a bots.yml keyword the user agent holds: the lowest, or the
+    # first one found when `lowest: false`.
+    def self.known_bot_rank(ua, lowest: true)
+      trie = known_bots_trie
+      found = nil
+      ua.bytesize.times do |start|
+        node = trie
+        position = start
+        while (node = node[ua.getbyte(position)])
+          rank = node[:rank]
+          if rank
+            return rank unless lowest
+
+            found = rank if found.nil? || rank < found
+          end
+          position += 1
+        end
+      end
+      found
+    end
+
+    # A Regexp.union of hundreds of words tries each of them at every
+    # position; nested by common prefix, the regexp follows a single branch
+    # per character.
+    def self.trie_source(words)
+      trie = {}
+      words.each do |word|
+        leaf = word.each_char.reduce(trie) { |node, char| node[char] ||= {} }
+        leaf[:end] = true
+      end
+      trie_node_source(trie) || "(?!)"
+    end
+
+    def self.trie_node_source(node)
+      # A shorter keyword matches wherever a longer one would.
+      return "" if node.key?(:end)
+
+      branches = node.map do |char, child|
+        Regexp.escape(char) + trie_node_source(child)
+      end
+      (branches.size == 1) ? branches.first : "(?:#{branches.join("|")})"
+    end
+    private_class_method :trie_source, :trie_node_source
+
+    def self.why?(ua)
+      ua = ua.downcase.strip
+      browser = Browser.new(ua)
+      matchers.find { |matcher| matcher.call(ua, browser) }
+    end
+
+    attr_reader :ua
+
+    def initialize(ua)
+      @ua = ua.downcase.strip
+    end
+
+    # A bot is shared by the browsers of its user agent: each answer holds
+    # until a matcher list or the bot list changes.
+    def bot?
+      generation = Cache.generation
+      unless @bot&.first == generation
+        @bot = [generation, !bot_exception? && detect_bot?]
+      end
+      @bot.last
+    end
+
+    def why?
+      self.class.matchers.find { |matcher| matcher.call(ua, self) }
+    end
+
+    def search_engine?
+      ua.match?(self.class.keywords_pattern(self.class.search_engines))
+    end
+
+    def name
+      return unless bot?
+
+      generation = Cache.generation
+      unless @name&.first == generation
+        @name = [generation, self.class.bots[known_bot_keyword] || GENERIC_NAME]
+      end
+      @name.last
+    end
+
+    # Test drivers have a category, but no bot? (bot_categories.yml).
+    def category
+      return if bot_exception?
+
+      keyword = bot? && known_bot_keyword
+      keyword ||= ua[self.class.keywords_pattern(self.class.category_by_keyword)]
+      self.class.category_by_keyword[keyword]
+    end
+
+    def ai?
+      AI_CATEGORIES.include?(category)
+    end
+
+    def user_triggered?
+      USER_TRIGGERED_CATEGORIES.include?(category)
+    end
+
+    # The owner publishing the IP ranges of the crawler this user agent claims
+    # to be (:google, :bing, :openai, :apple, :perplexity, :duckduckgo).
+    def owner
+      return unless bot?
+
+      @owner = IPRanges.owner(ua) unless defined?(@owner)
+      @owner
+    end
+
+    # Whether the IP belongs to the owner's published ranges. reverse_dns: true
+    # also asks the DNS for an IP outside them (Google, Bing and Apple): two
+    # network queries.
+    def verified?(ip, reverse_dns: false)
+      return false unless owner
+
+      IPRanges.include?(owner, ip) || (reverse_dns && IPRanges.verified_by_dns?(owner, ip))
+    end
+
+    private def known_bot_keyword
+      rank = self.class.known_bot_rank(ua)
+      self.class.known_bots_trie[:keywords][rank] if rank
+    end
+
+    private def bot_exception?
+      ua.match?(self.class.keywords_pattern(self.class.bot_exceptions))
+    end
+
+    private def detect_bot?
+      self.class.matchers.any? do |matcher|
+        matcher.call(ua, default_matchers.include?(matcher) ? nil : browser)
+      end
+    end
+
+    # The built-in matchers ignore the browser: building it would detect the
+    # user agent twice.
+    private def default_matchers
+      DEFAULT_MATCHERS
+    end
+
+    private def browser
+      @browser ||= Browser.new(ua)
+    end
+
+    private :ua
+  end
+end

data/lib/browser/browser.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+if Gem.loaded_specs.key?("browser")
+  raise LoadError, "agentry replaces the browser gem, they can't be loaded together: " \
+    "remove gem \"browser\" from the Gemfile (or whatever depends on it)"
+end
+
+require "json"
+require "pathname"
+
+require_relative "version"
+require_relative "detect_version"
+require_relative "accept_language"
+require_relative "base"
+require_relative "safari"
+require_relative "chrome"
+require_relative "internet_explorer"
+require_relative "firefox"
+require_relative "edge"
+require_relative "opera"
+require_relative "blackberry"
+require_relative "unknown"
+require_relative "phantom_js"
+require_relative "uc_browser"
+require_relative "nokia"
+require_relative "micro_messenger"
+require_relative "weibo"
+require_relative "qq"
+require_relative "alipay"
+require_relative "electron"
+require_relative "facebook"
+require_relative "otter"
+require_relative "instagram"
+require_relative "yandex"
+require_relative "sputnik"
+require_relative "snapchat"
+require_relative "duck_duck_go"
+require_relative "samsung_browser"
+require_relative "huawei_browser"
+require_relative "miui_browser"
+require_relative "maxthon"
+require_relative "sougou_browser"
+require_relative "google_search_app"
+require_relative "epiphany"
+
+require_relative "bot"
+require_relative "bot/empty_user_agent_matcher"
+require_relative "bot/keyword_matcher"
+require_relative "bot/known_bots_matcher"
+require_relative "bot/ip_ranges"
+
+require_relative "middleware"
+require_relative "platform"
+require_relative "device"
+require_relative "meta"
+require_relative "client_hints"
+require_relative "cloudflare"
+require_relative "detection"
+require_relative "cache"
+
+module Browser
+  EMPTY_STRING = ""
+
+  Bot::DEFAULT_MATCHERS =
+    [Bot::EmptyUserAgentMatcher, Bot::KnownBotsMatcher, Bot::KeywordMatcher]
+      .freeze
+
+  Error = Class.new(StandardError)
+
+  def self.root
+    @root ||= Pathname.new(File.expand_path("../..", __dir__))
+  end
+
+  # data/bots.json for "bots", written from bots.yml by script/build_data.rb: edit the YAML.
+  def self.load_data(name)
+    JSON.load_file(root.join("data", "#{name}.json"))
+  end
+
+  def self.eager_load!
+    [self, Platform, Device, Bot].each(&:matchers)
+    Meta.rules
+    AcceptLanguage.languages
+    Device::Samsung.names
+    Bot.known_bots_trie
+    [Bot.bot_exceptions, Bot.search_engines, Bot.category_by_keyword].each { |keywords| Bot.keywords_pattern(keywords) }
+    Bot::IPRanges.ranges_by_owner
+    nil
+  end
+
+  class << self
+    attr_accessor :user_agent_size_limit, :accept_language_size_limit
+    attr_reader :cache_size
+  end
+
+  self.user_agent_size_limit = 2048
+  self.accept_language_size_limit = 2048
+
+  def self.cache_size=(size)
+    unless size.is_a?(Integer) && size >= 0
+      raise ArgumentError, "cache_size must be an Integer >= 0, got #{size.inspect}"
+    end
+
+    @cache_size = size
+    Cache.clear
+  end
+
+  self.cache_size = 1000
+
+  # Hold the list of browser matchers.
+  # Order is important.
+  def self.matchers
+    @matchers ||= [
+      Nokia,
+      UCBrowser,
+      PhantomJS,
+      BlackBerry,
+      Opera,
+      Edge,
+      InternetExplorer,
+      Firefox,
+      Otter,
+      Epiphany,
+      Facebook,
+      Instagram,
+      Snapchat,
+      Weibo,
+      MicroMessenger, # must be placed before QQ
+      QQ,
+      Alipay,
+      Electron,
+      Yandex,
+      Sputnik,
+      DuckDuckGo,
+      SamsungBrowser,
+      HuaweiBrowser,
+      MiuiBrowser,
+      Maxthon,
+      SougouBrowser,
+      GoogleSearchApp,
+
+      # Must be last
+      Chrome,
+      Safari,
+      Unknown
+    ]
+  end
+
+  # A pattern that every user agent a built-in browser matches also matches:
+  # Browser.new skips, without building it, a browser whose pattern fails.
+  REQUIRED_PATTERN_BY_MATCHER = {
+    Nokia => /S40OviBrowser/,
+    UCBrowser => /UCBrowser/,
+    PhantomJS => /PhantomJS/,
+    BlackBerry => /BlackBerry|BB10/,
+    Opera => %r{Opera|OPR/},
+    Edge => %r{Edg|Trident/8},
+    InternetExplorer => %r{MSIE|Trident/},
+    Firefox => /Firefox|FxiOS/,
+    Otter => /Otter/,
+    Epiphany => /Epiphany/,
+    Facebook => /FBAV|FBAN/,
+    Instagram => /Instagram/,
+    Snapchat => /Snapchat/,
+    Weibo => /__weibo__/i,
+    MicroMessenger => /MicroMessenger/i,
+    QQ => %r{QQ/|QQBrowser}i,
+    Alipay => /AlipayClient/i,
+    Electron => /Electron/,
+    Yandex => /YaBrowser/,
+    Sputnik => /SputnikBrowser/,
+    DuckDuckGo => /DuckDuckGo/,
+    SamsungBrowser => /SamsungBrowser/,
+    HuaweiBrowser => /HuaweiBrowser/i,
+    MiuiBrowser => /MiuiBrowser/,
+    Maxthon => /Maxthon/i,
+    SougouBrowser => /(?i:SogouMobileBrowser)|\bSE\b/,
+    GoogleSearchApp => /GSA/
+  }.compare_by_identity.freeze
+
+  def self.new(user_agent, **)
+    user_agent ||= EMPTY_STRING
+    browser_class = Cache.cached_detection(user_agent)&.browser_class
+    return browser_class.new(user_agent, **) if browser_class
+
+    browser = detect(user_agent, **)
+    Cache.detection(user_agent).browser_class = browser.class
+    browser
+  end
+
+  def self.detect(user_agent, **)
+    matchers.each do |matcher_class|
+      required_pattern = REQUIRED_PATTERN_BY_MATCHER[matcher_class]
+      next if required_pattern && !required_pattern.match?(user_agent)
+
+      matcher = matcher_class.new(user_agent, **)
+      return matcher if matcher.match?
+    end
+  end
+  private_class_method :detect
+end

data/lib/browser/cache.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Browser
+  # The detections of the last user agents and the parsed Accept-Language
+  # headers, shared by the threads of a process. Reads take no lock: under the
+  # GVL a Hash read is atomic, and a thread that misses detects again. Writes
+  # do, for the Rubies without a GVL.
+  module Cache
+    @mutex = Mutex.new
+    @detection_by_user_agent = {}
+    @languages_by_header = {}
+    @generation = 0
+
+    def self.cached_detection(ua)
+      current_detection_by_user_agent[ua]
+    end
+
+    def self.detection(ua)
+      return Detection.new(ua) if Browser.cache_size.zero?
+
+      cached_detection(ua) || @mutex.synchronize do
+        ua = -ua
+        store(@detection_by_user_agent, ua, Detection.new(ua))
+      end
+    end
+
+    # A copy each time: callers may change the array they get.
+    def self.accept_language(header)
+      return AcceptLanguage.parse(header) if Browser.cache_size.zero?
+
+      languages = @languages_by_header[header] || @mutex.synchronize do
+        store(@languages_by_header, -header,
+          AcceptLanguage.parse(header).freeze)
+      end
+      languages.dup
+    end
+
+    # Bumped when a matcher list, the bot list or its exceptions change: the
+    # matchers are public arrays that apps extend at runtime.
+    def self.generation
+      return @generation if lists_unchanged?
+
+      @mutex.synchronize do
+        @generation += 1
+        @detection_by_user_agent.clear
+        remember_lists
+      end
+      @generation
+    end
+
+    def self.clear
+      @mutex.synchronize do
+        @detection_by_user_agent.clear
+        @languages_by_header.clear
+      end
+    end
+
+    # The bot list holds hundreds of entries: compared by identity and size,
+    # like Bot.keywords_pattern, not entry by entry on every request.
+    def self.lists_unchanged?
+      @browser_matchers == Browser.matchers &&
+        @platform_matchers == Platform.matchers &&
+        @device_matchers == Device.matchers &&
+        @bot_matchers == Bot.matchers &&
+        @bot_exceptions == Bot.bot_exceptions &&
+        @bots.equal?(Bot.bots) && @bots_size == Bot.bots.size
+    end
+
+    def self.remember_lists
+      @browser_matchers = Browser.matchers.dup
+      @platform_matchers = Platform.matchers.dup
+      @device_matchers = Device.matchers.dup
+      @bot_matchers = Bot.matchers.dup
+      @bot_exceptions = Bot.bot_exceptions.dup
+      @bots = Bot.bots
+      @bots_size = Bot.bots.size
+    end
+
+    def self.current_detection_by_user_agent
+      generation
+      @detection_by_user_agent
+    end
+
+    def self.store(value_by_key, key, value)
+      value_by_key.clear if value_by_key.size >= Browser.cache_size
+      value_by_key[key] = value
+    end
+
+    private_class_method :lists_unchanged?, :remember_lists,
+      :current_detection_by_user_agent, :store
+  end
+end

data/lib/browser/chrome.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Browser
+  class Chrome < Base
+    def id
+      :chrome
+    end
+
+    def name
+      "Chrome"
+    end
+
+    def full_version
+      # Each regex on its own line to enforce precedence.
+      ua[%r{Chrome/([\d.]+)}, 1] ||
+        ua[%r{CriOS/([\d.]+)}, 1] ||
+        ua[%r{Safari/([\d.]+)}, 1] ||
+        ua[%r{AppleWebKit/([\d.]+)}, 1] ||
+        "0.0"
+    end
+
+    def chromium_based?
+      true
+    end
+
+    def match?
+      ua.match?(/Chrome|CriOS/) &&
+        !ua.match?(/PhantomJS|FxiOS|ArchiveBot/) &&
+        !opera? &&
+        !edge? &&
+        !duck_duck_go? &&
+        !yandex? &&
+        !sputnik? &&
+        !samsung_browser? &&
+        !huawei_browser? &&
+        !miui_browser? &&
+        !maxthon? &&
+        !qq? &&
+        !sougou_browser? &&
+        !google_search_app?
+    end
+  end
+end

data/lib/browser/client_hints.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Browser
+  # The User-Agent Client Hints of Chromium browsers, read from a Rack env.
+  # Safari and Firefox send none: every answer is then nil.
+  class ClientHints
+    # A browser's hints are a few dozen bytes: longer values are ignored.
+    HEADER_SIZE_LIMIT_BYTES = 512
+    PLATFORM_CLASS_BY_NAME = {
+      "Android" => Platform::Android,
+      "Chrome OS" => Platform::ChromeOS,
+      "Linux" => Platform::Linux,
+      "macOS" => Platform::Mac,
+      "Windows" => Platform::Windows
+    }.freeze
+    # https://learn.microsoft.com/en-us/microsoft-edge/web-platform/how-to-detect-win11
+    WINDOWS_11_PLATFORM_VERSION = 13
+    BRAND_PATTERN = /"([^"\\]*)"\s*;\s*v\s*=\s*"([^"\\]*)"/
+    # GREASE brands, sent so that servers don't depend on the list's order.
+    GREASE_BRAND_PATTERN = /\ANot.A.Brand\z/i
+    QUOTED_STRING_PATTERN = /\A\s*"([^"\\]+)"\s*\z/
+    MOBILE_BY_VALUE = {"?1" => true, "?0" => false}.freeze
+
+    # Brand versions keyed by brand ("Google Chrome" => "142"), from Sec-CH-UA.
+    attr_reader :version_by_brand
+    # Sec-CH-UA-Platform ("Windows", "macOS", "Android"…).
+    attr_reader :platform
+    # Sec-CH-UA-Platform-Version ("15.0.0"), sent only to a site asking for it
+    # with an Accept-CH: Sec-CH-UA-Platform-Version response header.
+    attr_reader :platform_version
+
+    def initialize(headers)
+      @version_by_brand = parse_brands(header(headers, "HTTP_SEC_CH_UA"))
+      @mobile = MOBILE_BY_VALUE[header(headers, "HTTP_SEC_CH_UA_MOBILE")&.strip]
+      @platform = quoted_string(header(headers, "HTTP_SEC_CH_UA_PLATFORM"))
+      @platform_version = quoted_string(header(headers, "HTTP_SEC_CH_UA_PLATFORM_VERSION"))
+      freeze
+    end
+
+    # Sec-CH-UA-Mobile: nil when the browser did not send it.
+    def mobile?
+      @mobile
+    end
+
+    def platform_class
+      PLATFORM_CLASS_BY_NAME[platform]
+    end
+
+    # Windows 11 says "Windows NT 10.0" in its user agent: only its platform
+    # version tells it from Windows 10.
+    def windows11?
+      platform == "Windows" && platform_version.to_i >= WINDOWS_11_PLATFORM_VERSION
+    end
+
+    private def header(headers, name)
+      value = headers[name]
+      value if value.is_a?(String) && value.bytesize <= HEADER_SIZE_LIMIT_BYTES
+    end
+
+    private def parse_brands(value)
+      return EMPTY_HASH unless value
+
+      value.scan(BRAND_PATTERN).to_h.reject { |brand, _version| GREASE_BRAND_PATTERN.match?(brand) }.freeze
+    end
+
+    private def quoted_string(value)
+      value&.[](QUOTED_STRING_PATTERN, 1)
+    end
+
+    EMPTY_HASH = {}.freeze
+    NONE = new(EMPTY_HASH)
+  end
+end

data/lib/browser/cloudflare.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Browser
+  # What Cloudflare knows of the request, from the headers a Transform Rule
+  # sets (docs/guide.html#cloudflare) and its own CF-IPCountry. Each site names
+  # its headers in its rule: Browser::Cloudflare.header_names= changes them.
+  class Cloudflare
+    DEFAULT_HEADER_NAMES = {
+      verified_bot: "X-Bot-Verified",
+      bot_category: "X-Bot-Category",
+      bot_score: "X-Bot-Score",
+      country: "CF-IPCountry"
+    }.freeze
+
+    class << self
+      attr_reader :header_names, :env_key_by_signal
+    end
+
+    # Merged into the defaults: {bot_score: "X-Edge-Bot-Score"}.
+    def self.header_names=(names)
+      unknown_signals = names.keys - DEFAULT_HEADER_NAMES.keys
+      raise ArgumentError, "unknown Cloudflare signals: #{unknown_signals.inspect}" if unknown_signals.any?
+
+      @header_names = DEFAULT_HEADER_NAMES.merge(names).freeze
+      @env_key_by_signal = @header_names.transform_values { |name| "HTTP_#{name.upcase.tr("-", "_")}" }.freeze
+    end
+
+    self.header_names = {}
+
+    def initialize(headers)
+      @headers = headers
+    end
+
+    # cf.bot_management.verified_bot (cf.client.bot without Bot Management).
+    def verified_bot?
+      header(:verified_bot)&.casecmp?("true") || false
+    end
+
+    # cf.verified_bot_category, in bot.category's vocabulary: "Monitoring &
+    # Analytics" is :monitoring_and_analytics.
+    def bot_category
+      category = header(:bot_category)&.strip
+      return if category.nil? || category.empty?
+
+      category.downcase.gsub("&", "and").gsub(/[^a-z0-9]+/, "_").to_sym
+    end
+
+    # cf.bot_management.score, Bot Management only: 1 is a bot, 99 a human.
+    def bot_score
+      score = header(:bot_score)&.strip
+      score.to_i if score&.match?(/\A\d{1,2}\z/)
+    end
+
+    # The ISO 3166-1 alpha-2 country, "XX" when unknown, "T1" for Tor.
+    def country
+      country = header(:country)&.strip
+      country if country&.match?(/\A[A-Z0-9]{2}\z/)
+    end
+
+    private def header(signal)
+      value = @headers&.[](self.class.env_key_by_signal[signal])
+      value if value.is_a?(String)
+    end
+  end
+end