browser 2.7.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17e665a446ca2a9578cae49ad272bbbc596c29bf8518643cbcec39f2654e4bc4
-  data.tar.gz: 0dd132f6f496e0221417a5b474878823eed6b7e4eac51764f383cd2825cb1969
+  metadata.gz: 73c5a3410e573b333ce54118589c25ca868b6daa5df15e2e4962da120521139b
+  data.tar.gz: 7d47dbef918c0ec0176c59d81d592285e273d259769a80fc4edcafc335f69c9d
 SHA512:
-  metadata.gz: 2902444a66316eb79ccb9fd409d58d21ac764b6f22a86bb9a2af74791f3bd8c86301a7c7b8829fe17f14a24c0827b3584d0c0a7813f85c9a4022a179ea67db7a
-  data.tar.gz: 1282a961f637a98449d6425272abcd3f005c003908ce37e0954052b3b5fd8bc6bf3d5d75e3104e6d3705823e3abe60f9eb9fca7046bf94f1c16f625cc6a09b07
+  metadata.gz: ad5612535803d139af879d09312cdba4c77bc4c1351a483ed16fc677f4372177da92afeaea6b6fb87912e1cd9ae9450c49d97bc58cd494522ee9b206411e3b49
+  data.tar.gz: f1f8223336528e77c863843b918bd303be591e57b0b26c09bef24fe238406e3896637238b28b573792b8b8f25b20d1a4dbe3369dd30dd9e9671b8e702817b00e

data/.rubocop.yml

@@ -16,7 +16,7 @@ AllCops:
 Metrics/ClassLength:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 80
 
 Metrics/MethodLength:

data/CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## Unreleased
 
+- Add ArchiveTeam's ArchiveBot to the bot list.
+- Fix QQ Browser detection.
+- Update modern rules.
+- You can now define new bot matchers by adding a callable object to `Browser::Bot.matchers`.
+- Fix `browser.yandex?` and `browser.sputnik?`.
+- [BREAKING CHANGE] Removed methods to enable the bot's empty user agent detection (`Browser::Bot.detect_empty_ua!` and `Browser::Bot.detect_empty_ua?`).
+- [BREAKING CHANGE] Bot detection is now more aggressive by default. It matches empty user agents, anything that matches `crawl|fetch|search|monitoring|spider|bot`, and anything listed under https://github.com/fnando/browser/blob/master/bots.yml.
+- Add Jaunt to the bot list.
+
+## 2.7.1
+
 - Handle Snapchat user agents that have a space or an empty string instead of a slash before the version.
 - Fix iOS 10+ version detection.
 - Add fallback versions for instagram and snapchat to avoid NoMethodErrors on unexpected user agents.

data/README.md

@@ -58,7 +58,8 @@ browser.sputnik?
 browser.bot.name
 browser.bot.search_engine?
 browser.bot?
-Browser::Bot.why?(ua) # shows which user agent was the offender
+browser.bot.why? # shows which matcher detected this user agent as a bot.
+Browser::Bot.why?(ua)
 
 # Get device info
 browser.device
@@ -146,21 +147,21 @@ browser.mobile? #=> false
 
 ### What defines a modern browser?
 
-The current rules that define a modern browser are pretty loose:
+The current rules that define a modern browser are pretty loose.
 
-* Webkit
-* IE9+
-* Microsoft Edge
-* Firefox 17+
-* Firefox Tablet 14+
-* Opera 12+
+* Chrome 65+
+* Safari 10+
+* Firefox 52+
+* IE11+
+* Microsoft Edge 39+
+* Opera 50+
 
 You can define your own rules. A rule must be a proc/lambda or any object that implements the method === and accepts the browser object. To redefine all rules, clear the existing rules before adding your own.
 
 ```ruby
-# Only Chrome Canary is considered modern.
+# Only Google Chrome 79+ is considered modern.
 Browser.modern_rules.clear
-Browser.modern_rules << -> b { b.chrome? && b.version.to_i >= 37 }
+Browser.modern_rules << -> b { b.chrome? && b.version.to_i >= 79 }
 ```
 
 ### Rails integration
@@ -213,7 +214,7 @@ language.name
 #=> "English/United States"
 ```
 
-Result is always sorted in quality order from highest -> lowest. As per the HTTP spec:
+Result is always sorted in quality order from highest to lowest. As per the HTTP spec:
 
 - omitting the quality value implies 1.0.
 - quality value equal to zero means that is not accepted by the client.
@@ -266,10 +267,29 @@ browser.platform.ios?
 
 ### Bots
 
-Browser used to detect empty user agents as bots, but this behavior has changed. If you want to bring this detection back, you can activate it through the following call:
+The bot detection is quite aggressive. Anything that matches at least one of the following requirements will be considered a bot.
+
+- Empty user agent string
+- User agent that matches `/crawl|fetch|search|monitoring|spider|bot/`
+- Any known bot listed under [bots.yml](https://github.com/fnando/browser/blob/master/bots.yml)
+
+To add custom matchers, you can add a callable object to `Browser::Bot.matchers`. The following example matches everything that has a `externalhit` substring on it. The bot name will always be `General Bot`.
+
+```ruby
+Browser::Bot.matchers << ->(ua, _browser) { ua =~ /externalhit/i }
+```
+
+To clear all matchers, including the ones that are bundled, use `Browser::Bot.matchers.clear`. You can re-add built-in matchers by doing the following:
+
+```ruby
+Browser::Bot.matchers += Browser::Bot.default_matchers
+```
+
+To restore v2's bot detection, remove the following matchers:
 
 ```ruby
-Browser::Bot.detect_empty_ua!
+Browser::Bot.matchers.delete(Browser::Bot::KeywordMatcher)
+Browser::Bot.matchers.delete(Browser::Bot::EmptyUserAgentMatcher)
 ```
 
 ### Middleware

data/Rakefile

@@ -16,7 +16,14 @@ end
 require "rubocop/rake_task"
 desc "Run rubocop"
 task :rubocop do
-  RuboCop::RakeTask.new
+  RuboCop::RakeTask.new do |t|
+    t.options += %w[
+      --display-style-guide
+      --display-cop-names
+      --extra-details
+      --auto-correct
+    ]
+  end
 end
 
 desc "Run specs against all gemfiles"

data/bot_exceptions.yml

@@ -1,3 +1,4 @@
+---
 - ruby build
 - pinterest/android
 - pinterest/ios

data/bots.yml

@@ -1,3 +1,4 @@
+---
 200pleasebot: "200PleaseBot"
 360spider: "360Spider"
 abot: "CrawlDaddy, abot"
@@ -16,6 +17,7 @@ apis-google: APIs-Google
 appengine-google: "Google App Engine"
 applebot: "Apple Bot"
 archive.org_bot: "Internet Archive (archive.org)"
+archiveteam archivebot: "ArchiveTeam ArchiveBot"
 ask jeeves: "Ask Jeeves"
 asynchttpclient: "Java http and WebSocket client library"
 awe.sm: "Awe.sm URL expander"
@@ -109,6 +111,7 @@ insieve: "Insieve Bot"
 insitesbot: "Insitesbot"
 instapaper: "Instapaper"
 istellabot: "IstellaBot"
+jaunt: "Jaunt - Java Web Scraping & JSON Querying"
 jetslide: "Jetslide"
 jobseeker: "jobseeker.com.au/bot.html"
 jooble: "Jooble"
@@ -220,7 +223,6 @@ snapchat: "Snapchat"
 socialrank: "SocialRankIOBot"
 sogou: "Chinese search engine"
 spbot: "OpenLinkProfiler"
-spider: "generic web spider"
 spinn3r: "Spinn3r aggregator"
 sputnikbot: "SputnikBot"
 squider: "Squider"

data/browser.gemspec

@@ -29,6 +29,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rails"
   s.add_development_dependency "rake"
   s.add_development_dependency "rubocop"
-  s.add_development_dependency "rubocop-fnando"
+  s.add_development_dependency "rubocop-fnando", "~> 0.0.3"
   s.add_development_dependency "simplecov"
 end

data/lib/browser.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 
-require "browser/browser"
-require "browser/rails" if defined?(::Rails)
+require_relative "browser/browser"
+require_relative "browser/rails" if defined?(::Rails)

data/lib/browser/aliases.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "browser"
+require_relative "../browser"
 require "forwardable"
 
 module Browser

data/lib/browser/base.rb

@@ -151,12 +151,12 @@ module Browser
 
     # Detect if browser is Sputnik.
     def sputnik?(expected_version = nil)
-      Sputnik.new(ua) && detect_version?(full_version, expected_version)
+      Sputnik.new(ua).match? && detect_version?(full_version, expected_version)
     end
 
     # Detect if browser is Yandex.
     def yandex?(expected_version = nil)
-      Yandex.new(ua) && detect_version?(full_version, expected_version)
+      Yandex.new(ua).match? && detect_version?(full_version, expected_version)
     end
     alias_method :yandex_browser?, :yandex?
 

data/lib/browser/bot.rb

@@ -2,68 +2,76 @@
 
 module Browser
   class Bot
-    def self.detect_empty_ua!
-      @detect_empty_ua = true
+    GENERIC_NAME = "Generic Bot"
+
+    def self.matchers
+      @matchers ||= default_matchers
+    end
+
+    def self.default_matchers
+      [
+        EmptyUserAgentMatcher,
+        KnownBotsMatcher,
+        KeywordMatcher
+      ]
     end
 
-    def self.detect_empty_ua?
-      @detect_empty_ua
+    def self.load_yaml(path)
+      YAML.load_file(Browser.root.join(path))
     end
 
     def self.bots
-      @bots ||= YAML.load_file(Browser.root.join("bots.yml"))
+      @bots ||= load_yaml("bots.yml")
     end
 
     def self.bot_exceptions
-      @bot_exceptions ||= YAML
-                          .load_file(Browser.root.join("bot_exceptions.yml"))
+      @bot_exceptions ||= load_yaml("bot_exceptions.yml")
     end
 
     def self.search_engines
-      @search_engines ||= YAML
-                          .load_file(Browser.root.join("search_engines.yml"))
+      @search_engines ||= load_yaml("search_engines.yml")
     end
 
     def self.why?(ua)
-      downcased_ua = ua.downcase
-      bots.find {|key, _| downcased_ua.include?(key) }
+      ua = ua.downcase.strip
+      browser = Browser.new(ua)
+      matchers.find {|matcher| matcher.call(ua, browser) }
     end
 
-    attr_reader :ua
+    attr_reader :ua, :browser
 
     def initialize(ua)
-      @ua = ua
+      @ua = ua.downcase.strip
+      @browser = Browser.new(@ua)
     end
 
     def bot?
-      bot_with_empty_ua? || (!bot_exception? && detect_bot?)
+      !bot_exception? && detect_bot?
+    end
+
+    def why?
+      self.class.matchers.find {|matcher| matcher.call(ua, self) }
     end
 
     def search_engine?
-      self.class.search_engines.any? {|key, _| downcased_ua.include?(key) }
+      self.class.search_engines.any? {|key, _| ua.include?(key) }
     end
 
     def name
       return unless bot?
-      return "Generic Bot" if bot_with_empty_ua?
-
-      self.class.bots.find {|key, _| downcased_ua.include?(key) }.last
-    end
 
-    private def bot_with_empty_ua?
-      self.class.detect_empty_ua? && ua.strip == ""
+      self.class.bots.find {|key, _| ua.include?(key) }&.last || GENERIC_NAME
     end
 
     private def bot_exception?
-      self.class.bot_exceptions.any? {|key| downcased_ua.include?(key) }
+      self.class.bot_exceptions.any? {|key| ua.include?(key) }
     end
 
     private def detect_bot?
-      self.class.bots.any? {|key, _| downcased_ua.include?(key) }
+      self.class.matchers.any? {|matcher| matcher.call(ua, browser) }
     end
 
-    private def downcased_ua
-      @downcased_ua ||= ua.downcase
-    end
+    private :ua
+    private :browser
   end
 end

data/lib/browser/bot/empty_user_agent_matcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    class EmptyUserAgentMatcher
+      def self.call(ua, _browser)
+        ua == ""
+      end
+    end
+  end
+end

data/lib/browser/bot/keyword_matcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    class KeywordMatcher
+      def self.call(ua, _browser)
+        ua =~ /crawl|fetch|search|monitoring|spider|bot/
+      end
+    end
+  end
+end

data/lib/browser/bot/known_bots_matcher.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Browser
+  class Bot
+    class KnownBotsMatcher
+      def self.call(ua, _browser)
+        Browser::Bot.bots.any? {|key, _| ua.include?(key) }
+      end
+    end
+  end
+end

data/lib/browser/browser.rb

@@ -4,39 +4,42 @@ require "set"
 require "yaml"
 require "pathname"
 
-require "browser/version"
-require "browser/detect_version"
-require "browser/accept_language"
-require "browser/base"
-require "browser/safari"
-require "browser/chrome"
-require "browser/internet_explorer"
-require "browser/firefox"
-require "browser/edge"
-require "browser/opera"
-require "browser/blackberry"
-require "browser/generic"
-require "browser/phantom_js"
-require "browser/uc_browser"
-require "browser/nokia"
-require "browser/micro_messenger"
-require "browser/weibo"
-require "browser/qq"
-require "browser/alipay"
-require "browser/electron"
-require "browser/facebook"
-require "browser/otter"
-require "browser/instagram"
-require "browser/yandex"
-require "browser/sputnik"
-require "browser/snapchat"
+require_relative "version"
+require_relative "detect_version"
+require_relative "accept_language"
+require_relative "base"
+require_relative "safari"
+require_relative "chrome"
+require_relative "internet_explorer"
+require_relative "firefox"
+require_relative "edge"
+require_relative "opera"
+require_relative "blackberry"
+require_relative "generic"
+require_relative "phantom_js"
+require_relative "uc_browser"
+require_relative "nokia"
+require_relative "micro_messenger"
+require_relative "weibo"
+require_relative "qq"
+require_relative "alipay"
+require_relative "electron"
+require_relative "facebook"
+require_relative "otter"
+require_relative "instagram"
+require_relative "yandex"
+require_relative "sputnik"
+require_relative "snapchat"
 
-require "browser/bot"
-require "browser/middleware"
+require_relative "bot"
+require_relative "bot/empty_user_agent_matcher"
+require_relative "bot/keyword_matcher"
+require_relative "bot/known_bots_matcher"
 
-require "browser/platform"
-require "browser/device"
-require "browser/meta"
+require_relative "middleware"
+require_relative "platform"
+require_relative "device"
+require_relative "meta"
 
 module Browser
   EMPTY_STRING = ""
@@ -89,12 +92,12 @@ module Browser
   end
 
   modern_rules.tap do |rules|
-    rules << ->(b) { b.webkit? }
-    rules << ->(b) { b.firefox? && b.version.to_i >= 17 }
-    rules << ->(b) { b.ie? && b.version.to_i >= 9 && !b.compatibility_view? }
-    rules << ->(b) { b.edge? && !b.compatibility_view? }
-    rules << ->(b) { b.opera? && b.version.to_i >= 12 }
-    rules << ->(b) { b.firefox? && b.device.tablet? && b.platform.android? && b.version.to_i >= 14 } # rubocop:disable Metrics/LineLength
+    rules << ->(b) { b.chrome? && b.version.to_i >= 65 }
+    rules << ->(b) { b.safari? && b.version.to_i >= 10 }
+    rules << ->(b) { b.firefox? && b.version.to_i >= 52 }
+    rules << ->(b) { b.ie? && b.version.to_i >= 11 && !b.compatibility_view? }
+    rules << ->(b) { b.edge? && b.version.to_i >= 39 && !b.compatibility_view? }
+    rules << ->(b) { b.opera? && b.version.to_i >= 50 }
   end
 
   def self.new(user_agent, **kwargs)