agentry 3.0.0

data/lib/browser/accept_language.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Browser
+  class AcceptLanguage
+    def self.languages
+      @languages ||= Browser.load_data("languages")
+    end
+
+    def self.parse(accept_language)
+      return [] unless accept_language
+
+      languages = accept_language.split(",")
+      languages.map! { |string| new(string.squeeze(" ").strip) }
+      languages.reject! { |al| al.quality.zero? }
+      languages.sort_by.with_index { |al, idx| [-al.quality, idx] }
+    end
+
+    attr_reader :part
+
+    def initialize(part)
+      @part = part
+    end
+
+    def full
+      @full ||= [code, region].compact.join("-")
+    end
+
+    def name
+      self.class.languages[full] || self.class.languages[code]
+    end
+
+    def code
+      @code ||= part[/\A([^-;]+)/, 1]&.downcase
+    end
+
+    def region
+      @region ||= part[/\A(?:.*?)-([^;-]+)/, 1]&.upcase
+    end
+
+    def quality
+      @quality ||= begin
+        Float(quality_value || 1.0)
+      rescue ArgumentError
+        0.1
+      end
+    end
+
+    private def quality_value
+      qvalue = part[/;q=([\d.]+)/, 1]
+      qvalue = "0.0" if /\A0\.0?\z/.match?(qvalue)
+      qvalue = qvalue.squeeze(".") if qvalue
+      qvalue
+    end
+  end
+end

data/lib/browser/action_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "action_controller/railtie"
+
+module Browser
+  module ActionController
+    extend ActiveSupport::Concern
+
+    included do
+      helper_method(:browser) if respond_to?(:helper_method)
+    end
+
+    private def browser
+      @browser ||= Browser.new(
+        request.headers["User-Agent"],
+        accept_language: request.headers["Accept-Language"],
+        headers: request.env
+      )
+    end
+  end
+end

data/lib/browser/aliases.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "../browser"
+require "forwardable"
+
+module Browser
+  module Aliases
+    PLATFORM_ALIASES = %w[
+      adobe_air? android? blackberry? chrome_os? firefox_os? ios? ios_app?
+      ios_webview? linux? mac? windows10? windows7? windows8? windows8_1?
+      windows? windows_mobile? windows_phone? windows_rt?
+      windows_touchscreen_desktop? windows_vista? windows_wow64? windows_x64?
+      windows_x64_inclusive? windows_xp?
+    ].freeze
+
+    DEVICE_ALIASES = %w[
+      blackberry_playbook? console? ipad? iphone? ipod_touch? kindle?
+      kindle_fire? mobile? nintendo? nintendo_switch? nintendo_wii?
+      nintendo_wiiu? playbook? playstation3? playstation4? playstation?
+      playstation_vita? ps3? ps4? psp? psp_vita? silk? surface? tablet? tv?
+      vita? wii? wiiu? xbox? xbox_360? xbox_one?
+    ].freeze
+
+    def self.included(target)
+      target.class_eval do
+        extend Forwardable
+
+        def_delegators :platform, *PLATFORM_ALIASES
+        def_delegators :device, *DEVICE_ALIASES
+      end
+    end
+  end
+end

data/lib/browser/alipay.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Browser
+  class Alipay < Base
+    def id
+      :alipay
+    end
+
+    def name
+      "Alipay"
+    end
+
+    def full_version
+      ua[%r{(?:AlipayClient)/([\d.]+)}i, 1] || "0.0"
+    end
+
+    def match?
+      ua.match?(/AlipayClient/i)
+    end
+  end
+end

data/lib/browser/base.rb

@@ -0,0 +1,307 @@
+# frozen_string_literal: true
+
+module Browser
+  class Base
+    include DetectVersion
+
+    attr_reader :ua
+
+    # headers: the Rack env (request.env), for the client hints and the
+    # Cloudflare headers.
+    def initialize(ua, accept_language: nil, headers: nil)
+      validate_size(:user_agent, ua.to_s)
+
+      @ua = ua
+      @accept_language_raw = accept_language.to_s
+      @headers = headers
+    end
+
+    # Return a meta info about this browser.
+    def meta
+      Meta.get(self)
+    end
+
+    # Return an array with all preferred languages that this browser accepts.
+    def accept_language
+      @accept_language ||= begin
+        validate_size(:accept_language, @accept_language_raw)
+        Cache.accept_language(@accept_language_raw)
+      end
+    end
+
+    alias_method :to_a, :meta
+
+    # Return meta representation as string.
+    def to_s
+      meta.to_a.join(" ")
+    end
+
+    def version
+      full_version.split(".").first
+    end
+
+    # Return the platform.
+    def platform
+      @platform ||= detection.platform.with_client_hints(client_hints)
+    end
+
+    def client_hints
+      @client_hints ||= @headers ? ClientHints.new(@headers) : ClientHints::NONE
+    end
+
+    def cloudflare
+      @cloudflare ||= Cloudflare.new(@headers)
+    end
+
+    # Return the bot info.
+    def bot
+      @bot ||= detection.bot
+    end
+
+    # Detect if current user agent is from a bot.
+    def bot?
+      bot.bot?
+    end
+
+    # Return the device info.
+    def device
+      @device ||= detection.device.with_client_hints(client_hints, platform:)
+    end
+
+    # Detect if browser is Microsoft Internet Explorer.
+    def ie?(expected_version = nil)
+      instance_of?(InternetExplorer) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Microsoft Edge.
+    def edge?(expected_version = nil)
+      instance_of?(Edge) && detect_version?(full_version, expected_version)
+    end
+
+    def compatibility_view?
+      false
+    end
+
+    def msie_full_version
+      "0.0"
+    end
+
+    def msie_version
+      "0"
+    end
+
+    # Detect if browser is Instagram.
+    def instagram?(expected_version = nil)
+      instance_of?(Instagram) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Snapchat.
+    def snapchat?(expected_version = nil)
+      instance_of?(Snapchat) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser if Facebook.
+    def facebook?(expected_version = nil)
+      instance_of?(Facebook) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Otter.
+    def otter?(expected_version = nil)
+      instance_of?(Otter) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is WebKit-based.
+    def webkit?(expected_version = nil)
+      ua.match?(/AppleWebKit/i) &&
+        (!edge? || chromium_based?) &&
+        detect_version?(webkit_full_version, expected_version)
+    end
+
+    # Detect if browser is QuickTime
+    def quicktime?(expected_version = nil)
+      ua.match?(/QuickTime/i) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Apple CoreMedia.
+    def core_media?(expected_version = nil)
+      ua.include?("CoreMedia") && detect_version?(full_version,
+        expected_version)
+    end
+
+    # Detect if browser is PhantomJS
+    def phantom_js?(expected_version = nil)
+      instance_of?(PhantomJS) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Safari.
+    def safari?(expected_version = nil)
+      instance_of?(Safari) && detect_version?(full_version, expected_version)
+    end
+
+    def safari_webapp_mode?
+      (device.ipad? || device.iphone?) && ua.include?("AppleWebKit")
+    end
+
+    # Detect if browser is Firefox.
+    def firefox?(expected_version = nil)
+      instance_of?(Firefox) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Chrome.
+    def chrome?(expected_version = nil)
+      instance_of?(Chrome) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Opera.
+    def opera?(expected_version = nil)
+      instance_of?(Opera) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Sputnik.
+    def sputnik?(expected_version = nil)
+      instance_of?(Sputnik) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Yandex.
+    def yandex?(expected_version = nil)
+      instance_of?(Yandex) && detect_version?(full_version, expected_version)
+    end
+    alias_method :yandex_browser?, :yandex?
+
+    # Detect if browser is UCBrowser.
+    def uc_browser?(expected_version = nil)
+      instance_of?(UCBrowser) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Nokia S40 Ovi Browser.
+    def nokia?(expected_version = nil)
+      instance_of?(Nokia) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is MicroMessenger.
+    def micro_messenger?(expected_version = nil)
+      instance_of?(MicroMessenger) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    alias_method :wechat?, :micro_messenger?
+
+    def weibo?(expected_version = nil)
+      instance_of?(Weibo) && detect_version?(full_version, expected_version)
+    end
+
+    def alipay?(expected_version = nil)
+      instance_of?(Alipay) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Opera Mini.
+    def opera_mini?(expected_version = nil)
+      ua.include?("Opera Mini") && detect_version?(full_version,
+        expected_version)
+    end
+
+    # Detect if browser is DuckDuckGo.
+    def duck_duck_go?(expected_version = nil)
+      ua.include?("DuckDuckGo") && detect_version?(full_version,
+        expected_version)
+    end
+
+    # Detect if browser is Samsung.
+    def samsung_browser?(expected_version = nil)
+      ua.include?("SamsungBrowser") && detect_version?(full_version,
+        expected_version)
+    end
+
+    # Detect if browser is Huawei.
+    def huawei_browser?(expected_version = nil)
+      instance_of?(HuaweiBrowser) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Xiaomi Miui.
+    def miui_browser?(expected_version = nil)
+      instance_of?(MiuiBrowser) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Maxthon.
+    def maxthon?(expected_version = nil)
+      instance_of?(Maxthon) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is QQ.
+    def qq?(expected_version = nil)
+      instance_of?(QQ) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Sougou.
+    def sougou_browser?(expected_version = nil)
+      instance_of?(SougouBrowser) &&
+        detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Google Search App
+    def google_search_app?(expected_version = nil)
+      ua.include?("GSA") && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if browser is Chromium-based.
+    def chromium_based?
+      false
+    end
+
+    def webkit_full_version
+      ua[%r{AppleWebKit/([\d.]+)}, 1] || "0.0"
+    end
+
+    def known?
+      !unknown?
+    end
+
+    def unknown?
+      id == :unknown_browser
+    end
+
+    # Detect if browser is a proxy browser.
+    def proxy?
+      nokia? || uc_browser? || opera_mini?
+    end
+
+    # Detect if the browser is Electron.
+    def electron?(expected_version = nil)
+      instance_of?(Electron) && detect_version?(full_version, expected_version)
+    end
+
+    # Detect if the browser is Epiphany.
+    def epiphany?(expected_version = nil)
+      instance_of?(Epiphany) && detect_version?(full_version, expected_version)
+    end
+
+    private def detection
+      @detection ||= Cache.detection(ua)
+    end
+
+    private def validate_size(subject, input)
+      actual_bytesize = input.bytesize
+      size_limit =
+        if subject == :user_agent
+          Browser.user_agent_size_limit
+        else
+          Browser.accept_language_size_limit
+        end
+
+      return if actual_bytesize < size_limit
+
+      raise Error,
+        "#{subject} cannot be larger than #{size_limit} bytes; " \
+        "actual size is #{actual_bytesize} bytes"
+    end
+  end
+end

data/lib/browser/blackberry.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Browser
+  class BlackBerry < Base
+    def id
+      :blackberry
+    end
+
+    def name
+      "BlackBerry"
+    end
+
+    def full_version
+      ua[%r{BlackBerry[\da-z]+/([\d.]+)}, 1] ||
+        ua[%r{Version/([\d.]+)}, 1] ||
+        "0.0"
+    end
+
+    def match?
+      ua.match?(/BlackBerry|BB10/)
+    end
+  end
+end

data/lib/browser/bot/empty_user_agent_matcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    class EmptyUserAgentMatcher
+      def self.call(ua, _browser)
+        ua == ""
+      end
+    end
+  end
+end

data/lib/browser/bot/ip_ranges.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+module Browser
+  class Bot
+    # The IP ranges crawler owners publish, from bot_ip_ranges.yml (rewritten
+    # weekly by script/sync_ip_ranges.rb) or fetched again by `refresh`.
+    module IPRanges
+      GOOGLE_RANGES_URL = "https://developers.google.com/static/crawling/ipranges"
+      SOURCES = {
+        google: {
+          keywords: %w[
+            adsbot-google apis-google feedfetcher-google google-cloudvertexbot google-inspectiontool
+            google-read-aloud google-safety google-site-verification googleagent-urlcontext googlebot
+            googleother googleproducer mediapartners-google storebot-google
+          ],
+          # https://developers.google.com/search/docs/crawling-indexing/verifying-googlebot
+          domains: %w[googlebot.com google.com googleusercontent.com],
+          urls: %w[common-crawlers.json special-crawlers.json user-triggered-fetchers.json
+            user-triggered-fetchers-google.json user-triggered-agents.json].map { |file| "#{GOOGLE_RANGES_URL}/#{file}" }
+        },
+        bing: {
+          keywords: %w[adidxbot bingbot bingpreview microsoftpreview msnbot],
+          domains: %w[search.msn.com],
+          urls: %w[https://www.bing.com/toolbox/bingbot.json]
+        },
+        openai: {
+          keywords: %w[chatgpt-user gptbot oai-searchbot],
+          domains: [],
+          urls: %w[https://openai.com/gptbot.json https://openai.com/searchbot.json
+            https://openai.com/chatgpt-user.json]
+        },
+        apple: {
+          keywords: %w[applebot],
+          domains: %w[applebot.apple.com],
+          urls: %w[https://search.developer.apple.com/applebot.json]
+        },
+        perplexity: {
+          keywords: %w[perplexity-user perplexitybot],
+          domains: [],
+          urls: %w[https://www.perplexity.ai/perplexitybot.json https://www.perplexity.ai/perplexity-user.json]
+        },
+        duckduckgo: {
+          keywords: %w[duckassistbot duckduckbot],
+          domains: [],
+          urls: %w[https://duckduckgo.com/duckduckbot.json https://duckduckgo.com/duckassistbot.json]
+        }
+      }.freeze
+      FETCH_TIMEOUT_SECONDS = 10
+      # Owners move their files: Google's old URLs redirect to the new ones.
+      MAX_REDIRECTS = 3
+      DNS_TIMEOUT_SECONDS = 2
+
+      # The owner whose crawler the lowercase user agent claims to be.
+      def self.owner(ua)
+        SOURCES.each_key.find do |owner|
+          SOURCES[owner][:keywords].any? { |keyword| ua.include?(keyword) }
+        end
+      end
+
+      def self.include?(owner, ip)
+        address = parse(ip)
+        return false unless address
+
+        starts, ends = ranges_by_owner.dig(owner, address.family)
+        return false unless starts
+
+        number = address.to_i
+        index = (starts.bsearch_index { |start| start > number } || starts.size) - 1
+        index >= 0 && number <= ends[index]
+      end
+
+      # Google's documented method: the reverse lookup names one of the owner's
+      # domains, and that name resolves back to the IP. Two DNS queries.
+      def self.verified_by_dns?(owner, ip)
+        address = parse(ip)
+        domains = SOURCES.dig(owner, :domains)
+        return false unless address && domains&.any?
+
+        require "resolv"
+        Resolv::DNS.open do |dns|
+          dns.timeouts = DNS_TIMEOUT_SECONDS
+          host = dns.getname(address.to_s).to_s.downcase
+          domains.any? { |domain| host == domain || host.end_with?(".#{domain}") } &&
+            dns.getaddresses(host).any? { |resolved| IPAddr.new(resolved.to_s) == address }
+        end
+      rescue Resolv::ResolvError, Resolv::ResolvTimeout, SystemCallError
+        false
+      end
+
+      # Prefixes ("66.249.64.0/27"), keyed by owner.
+      def self.prefixes_by_owner
+        @prefixes_by_owner ||= Browser.load_data("bot_ip_ranges").transform_keys(&:to_sym)
+      end
+
+      def self.ranges_by_owner
+        @ranges_by_owner ||= build_ranges(prefixes_by_owner)
+      end
+
+      # For long-lived processes: fetches every owner's ranges again and swaps
+      # them in at once. Raises Browser::Error and keeps the current ranges when
+      # a source fails.
+      def self.refresh
+        load(fetch)
+      end
+
+      def self.load(prefixes_by_owner)
+        ranges_by_owner = build_ranges(prefixes_by_owner)
+        @prefixes_by_owner = prefixes_by_owner
+        @ranges_by_owner = ranges_by_owner
+      end
+
+      def self.fetch
+        SOURCES.to_h do |owner, source|
+          addresses = source[:urls].flat_map { |url| fetch_prefixes(url) }.map do |prefix|
+            IPAddr.new(prefix)
+          rescue ArgumentError
+            raise Error, "#{owner}: invalid prefix #{prefix.inspect}"
+          end
+          sorted = addresses.uniq.sort_by { |address| [address.family, address.to_i, address.prefix] }
+          [owner, sorted.map { |address| "#{address}/#{address.prefix}" }]
+        end
+      end
+
+      def self.fetch_prefixes(url)
+        require "json"
+        require "net/http"
+        uri = URI(url)
+        response = nil
+        (MAX_REDIRECTS + 1).times do
+          response = Net::HTTP.start(uri.host, uri.port, use_ssl: true,
+            open_timeout: FETCH_TIMEOUT_SECONDS, read_timeout: FETCH_TIMEOUT_SECONDS) do |http|
+            http.get(uri.request_uri, "User-Agent" => "Agentry/#{VERSION} (+https://github.com/levups/agentry)")
+          end
+          break unless response.is_a?(Net::HTTPRedirection)
+
+          uri = uri.merge(response["Location"])
+        end
+        raise Error, "#{url}: HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+        prefixes = JSON.parse(response.body).fetch("prefixes").map do |prefix|
+          prefix["ipv4Prefix"] || prefix["ipv6Prefix"]
+        end
+        raise Error, "#{url}: no prefixes" if prefixes.empty?
+
+        prefixes
+      rescue JSON::ParserError, KeyError, Timeout::Error, IOError, SystemCallError, OpenSSL::SSL::SSLError => exception
+        raise Error, "#{url}: #{exception.message}"
+      end
+
+      # Per owner and address family, the merged ranges as two sorted arrays of
+      # integers, first and last addresses, for a binary search.
+      def self.build_ranges(prefixes_by_owner)
+        prefixes_by_owner.to_h do |owner, prefixes|
+          ranges_by_family = prefixes.map { |prefix| IPAddr.new(prefix) }.group_by(&:family)
+          [owner.to_sym, ranges_by_family.transform_values { |addresses| merge(addresses) }]
+        end
+      end
+
+      def self.merge(addresses)
+        ranges = addresses.map do |address|
+          range = address.to_range
+          [range.begin.to_i, range.end.to_i]
+        end
+        ranges.sort!
+        starts = []
+        ends = []
+        ranges.each do |first, last|
+          if ends.any? && first <= ends.last + 1
+            ends[-1] = last if last > ends.last
+          else
+            starts << first
+            ends << last
+          end
+        end
+        [starts, ends]
+      end
+
+      # A forged or malformed X-Forwarded-For must not raise in a request.
+      def self.parse(ip)
+        ip = ip.to_s
+        return if ip.include?("/")
+
+        address = IPAddr.new(ip)
+        address.ipv4_mapped? ? address.native : address
+      rescue ArgumentError
+        nil
+      end
+
+      private_class_method :fetch_prefixes, :build_ranges, :merge, :parse
+    end
+  end
+end

data/lib/browser/bot/keyword_matcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    class KeywordMatcher
+      def self.call(ua, _browser)
+        ua.match?(/crawl|fetch|search|monitoring|spider|bot/)
+      end
+    end
+  end
+end

data/lib/browser/bot/known_bots_matcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    class KnownBotsMatcher
+      def self.call(ua, _browser)
+        !Browser::Bot.known_bot_rank(ua, lowest: false).nil?
+      end
+    end
+  end
+end