browser 2.5.2 → 2.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 43aabd1ab4b633828eb6be0c7aad0fa1db432e05
-  data.tar.gz: 47b1576b89b96fad8a74df096c599179de94a714
+SHA256:
+  metadata.gz: 293548a9eea57000babf98f31dd4fbb5652886d9af4d0fea9be163e58d041c10
+  data.tar.gz: 0e5c8ae27a8b45b751c1b8c2a40aa0daf37a6a7c6547621a2dee348a6f207216
 SHA512:
-  metadata.gz: 40c7b4dc28bf96eff5ee645cf0acc1377bd2ead45741ae6e95b9a6edff0c686118bc6cd881e7ca07e49f91d5064d6e096a3bd94d2fee45e2ec22c42459b07194
-  data.tar.gz: 149613ad3ceb89275338691a152d2db7fb2b48e0d5cc9d8e31e6db19e1dc6a1076c269bd01156e50682812432e980b251e8c5959cdd95150b9dd41e7b929ba8d
+  metadata.gz: 708cd426295863ce846e4b7934ac6fa7d2716c3a0e2bf136dab0f4f79dde7648a3a41ad2dca3565bde09abe0658b08e2b5dce184d371a449f9ecbceb7a152629
+  data.tar.gz: 2d6c9699c81fe9bd5be50089682524c12076b3962bcc2936d9592fd93d5c4cb467dcd8ece8c78cd9252bc3395884b8b3bace3daedfdfa8742a74c51b2b280a92

data/.travis.yml

@@ -2,12 +2,12 @@ language: ruby
 sudo: false
 cache: bundler
 rvm:
-  - '2.4.1'
-  - '2.3.4'
-  - '2.2.7'
+  - '2.4.2'
+  - '2.3.5'
 gemfile:
   - Gemfile
   - gemfiles/rails4.gemfile
+  - gemfiles/rails5.gemfile
 script: bundle exec rake
 notifications:
   email: false

data/CHANGELOG.md

@@ -1,14 +1,31 @@
 # Changelog
 
+## v2.5.3
+
+- Add Google Site Verification to the bot list.
+- Handle invalid quality values that look like numbers.
+- Add Barkrowler bot.
+- Add AlwaysOnline bot: CloudFlare.
+- Add News aggregator crawler: AndersPink, BuzzBot.
+- Add Domain crawler: CipaCrawler.
+- Add Job bot: JobSeeker's.
+- Add Apparel crawler: TeeRaid.
+- Add Search engine crawler: SemanticBot, Mappy.
+- Add Copyright crawler: Copypants' BotPants.
+- Add SEO bots: SEOdiver, SeoAudit, WebCeo.
+- Add Woriobot from Zite.
+- Add BUbiNG bot.
+- Add Paessler bot.
+
 ## v2.5.2
 
-- Add COMODO SSL Checker bot
-- Add Swiftype bot
-- Add WhatsApp detection
+- Add COMODO SSL Checker bot.
+- Add Swiftype bot.
+- Add WhatsApp detection.
 
 ## v2.5.1
 
-- Add Android Oreo detection
+- Add Android Oreo detection.
 
 ## v2.5.0
 
@@ -74,4 +91,4 @@
 - `Browser#bot` now returns a `Browser::Bot` instance.
 - Safari running as web app mode is not recognized as Safari anymore.
 - ruby-2.3+ will always activate frozen strings.
-- [List of all commits since last release](https://github.com/fnando/browser/compare/v1.1.0...v2.0.0)
+- [List of all commits since last release](https://github.com/fnando/browser/compare/v1.1.0...v2.0.0).

data/Rakefile

@@ -9,7 +9,6 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList["test/**/*_test.rb"]
   t.verbose = true
   t.warning = false
-  t.ruby_opts = %w[-rubygems]
 end
 
 require "rubocop/rake_task"

data/bots.yml

@@ -8,6 +8,7 @@ adsbot-google: "Google Adwords"
 advbot: "AdvBot"
 ahrefsbot: "Ahrefs backlinks research tool"
 alexa: "Alexa Crawler"
+anderspink: "AndersPinkBot"
 apache-httpclient: "Java http library"
 apachebench: "ApacheBench (ab)"
 apis-google: APIs-Google
@@ -18,6 +19,7 @@ ask jeeves: "Ask Jeeves"
 asynchttpclient: "Java http and WebSocket client library"
 awe.sm: "Awe.sm URL expander"
 baidu: "Baidu"
+barkrowler: "Barkrowler"
 bdcbot: "Big Data Corp"
 bingbot: "Microsoft Bing"
 bingpreview: "Microsoft Bing preview"
@@ -25,18 +27,23 @@ bitlybot: "bit.ly bot"
 blekkobot: "Blekkobot"
 blexbot: "BLEXBot (webmeup)"
 bot@linkfluence.net: "Linkfluence bot"
+bubing: "BUbiNG"
 bufferbot: "BufferBot"
 buibui-checkbot: "buibui"
 butterfly: "Topsy Labs"
+buzzbot: "Buzzbot"
 buzztalk: "buzztalk"
 catchbot: "CatchBot (catchbot.com)"
 check_http: "Nagios monitor"
+cipacrawler: "CipaCrawler"
 cliqzbot: "Cliqzbot"
+cloudflare: "CloudFlare-AlwaysOnline"
 cmradar/0.1: "CMRadar/0.1"
 coldfusion: "ColdFusion http library"
 commoncrawl: "CCBot"
 comodo ssl checker: 'COMODO SSL Checker'
 comodo-webinspector-crawler: "Comodo"
+copypants: "BotPants"
 crowsnest: "Crowsnest"
 curabot: "cura.yt"
 curl: "curl unix CLI http client"
@@ -74,6 +81,7 @@ gimmeusabot: "Gimme60 (gimme60.com)"
 go http package: "Go http library"
 google page speed insights: "Google Page Speed Insights"
 google Web Preview: "Google Instant Previews crawler"
+google-site-verification: Google-Site-Verification
 google-structured-data-testing-tool: "Google-StructuredDataTestingTool"
 google-structureddatatestingtool: "Google-StructuredDataTestingTool"
 googlebot: "Google Bot"
@@ -99,6 +107,7 @@ jack: "jack"
 jakarta commons: "Jakarta Commons HttpClient"
 java: "Generic Java http library"
 jetslide: "Jetslide"
+jobseeker: "jobseeker.com.au/bot.html"
 js-kit: "URL resolver"
 kemvibot: "Kemvi"
 kimengi: "Kimengi Bot"
@@ -122,6 +131,7 @@ lumibot: "Lumibot"
 lwp-trivial: "Another Perl library"
 magpie-crawler: "magpie-crawler"
 mail.ru_bot: "Mail.ru Bot"
+mappydata: "Mappy"
 meanpathbot: "meanpath"
 mediapartners-google: "Google Adsense bot"
 megaindex.ru: "MegaIndex"
@@ -148,6 +158,7 @@ ning: "NING - Yet Another Twitter Swarmer"
 nutch: "Apache search spider"
 openhosebot: "OpenHoseBot"
 orangebot: "OrangeBot"
+paessler: "paessler.com - PRTG Network Monitor"
 pagesinventory: "pagesinventory.com"
 panopta: "Monitoring service"
 paperlibot: "PaperLi"
@@ -180,7 +191,10 @@ ruby: "Ruby"
 scrapy: "Scrapy"
 screaming frog seo spider: Screaming Frog SEO Spider
 searchmetricsbot: "SearchmetricsBot"
+semanticbot: "Semanticbot"
 semrushbot: "SEO analysis bot"
+seo-audit: "seo-audit-check-bot"
+seodiver: "SEOdiver"
 seokicks: "SEOKicks"
 seznambot: "SeznamBot"
 shopwiki: "ShopWiki"
@@ -203,6 +217,7 @@ squider: "Squider"
 statuscake: "StatusCake"
 stripe: "Stripe"
 swiftbot: "Swiftype Bot"
+teeraid: "TeeRaidBot"
 test certificate info: "C http library?"
 tineye: "TinEye Bot"
 traackr: "Traackr Bot"
@@ -228,11 +243,13 @@ vrcrawler: "Venture Radar"
 wasalive-bot: "Wasalive Bots"
 watchsumo: "WatchSumo"
 wbsearchbot: "Ware Bay Best Buys"
+webceo: "online-webceo-bot"
 webscout: "Webscout"
 wesee: "WeSEE"
 wget: "wget unix CLI http client"
 whatsapp: "WhatsApp"
 wordpress: "WordPress spider"
+woriobot: "woriobot"
 wormly: "WormlyBot"
 wotbox: "Wotbox"
 xenu link sleuth: "Xenu Link Sleuth"
@@ -241,6 +258,7 @@ xovibot: "XoviBot"
 yacybot: "YaCy"
 yahoo-ad-monitoring: "Yahoo Ad monitoring"
 yandex: "Yandex"
+yanga: "Yanga WorldSearch Bot"
 yeti: "Naver Corp"
 yourls: "YOURLS"
 zelist.ro: "feed parser"

data/gemfiles/rails5.gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+gemspec path: ".."
+
+gem "rails", '~> 5.0'

data/lib/browser/accept_language.rb

@@ -47,14 +47,20 @@ module Browser
     end
 
     def quality
-      @quality ||= Float(quality_value || 1.0)
+      @quality ||= begin
+        Float(quality_value || 1.0)
+      rescue ArgumentError
+        0.1
+      end
     end
 
     private
 
     def quality_value
       qvalue = part[/;q=([\d.]+)/, 1]
-      qvalue =~ /\A0\.0?\z/ ? "0.0" : qvalue
+      qvalue = qvalue =~ /\A0\.0?\z/ ? "0.0" : qvalue
+      qvalue = qvalue.gsub(/\.+/, ".") if qvalue
+      qvalue
     end
   end
 end

data/lib/browser/rails.rb

@@ -9,9 +9,13 @@ module Browser
     config.browser = ActiveSupport::OrderedOptions.new
 
     initializer "browser" do
-      ::ActionController::Base.send :include, Browser::ActionController
-      Browser::Middleware::Context.send :include,
-                                        Browser::Middleware::Context::Additions
+      ActiveSupport.on_load(:action_controller) do
+        ::ActionController::Base.send :include, Browser::ActionController
+        Browser::Middleware::Context.send(
+          :include,
+          Browser::Middleware::Context::Additions
+        )
+      end
     end
   end
 end

data/lib/browser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Browser
-  VERSION = "2.5.2".freeze
+  VERSION = "2.5.3".freeze
 end

data/test/ua_bots.yml

@@ -1,13 +1,20 @@
 ADLXBOT: 'Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)'
+ANDERSPINK: 'Mozilla/5.0 (compatible; AndersPinkBot/1.0; +http://anderspink.com/bot.html)'
 APIS_GOOGLE: 'APIs-Google; (+https://developers.google.com/webmasters/APIs-Google.html)'
 APPLE_BOT: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1)'
 ASK: 'Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)'
 AWS_ELB: ELB-HealthChecker/1.0
 BAIDU: 'Baiduspider+(+http://www.baidu.com/search/spider.htm)'
+BARKROWLER: 'Barkrowler/0.7 (+http://www.exensa.com/crawl)'
 BINGBOT: 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'
 BINGPREVIEW: 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b'
+BUBING: 'BUbiNG (+http://law.di.unimi.it/BUbiNG.html)'
+BUZZBOT: 'Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)'
+CIPACRAWLER: 'CipaCrawler/3.0 (info@domaincrawler.com; http://www.domaincrawler.com/www.example.com)'
+CLOUDFLARE: "Mozilla/5.0 (compatible; CloudFlare-AlwaysOnline/1.0; +http://www.cloudflare.com/always-online) AppleWebKit/534.34"
 COMMONCRAWL: 'CCBot/2.0 (http://commoncrawl.org/faq/)'
 COMODO_SSL_CHECKER: 'COMODO SSL Checker'
+COPYPANTS: 'Mozilla/5.0 (compatible; BotPants/1.0; Linux; +info@copypants.com) KHTML/3.5.5 (like Gecko)'
 DAUMOA: Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0
 DOMAINAREANIMATOR: 'Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com'
 DOT_BOT: 'Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)'
@@ -15,14 +22,17 @@ DUCKDUCKGO: 'DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)'
 FACEBOOK_BOT: 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)'
 GOOGLE_BOT: 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
 GOOGLE_PAGE_SPEED_INSIGHTS: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML, like Gecko; Google Page Speed Insights) Chrome/22.0.1229 Safari/537.4'
+GOOGLE_SITE_VERIFICATION: Mozilla/5.0 (compatible; Google-Site-Verification/1.0)
 GOOGLE_STACKDRIVER_UPTIME_CHECKS: 'GoogleStackdriverMonitoring-UptimeChecks'
 GOOGLE_STRUCTURED_DATA_TESTING_TOOL2: 'Mozilla/5.0 (compatible; Google-Structured-Data-Testing-Tool +http://developers.google.com/structured-data/testing-tool/)'
 GOOGLE_STRUCTURED_DATA_TESTING_TOOL: 'Mozilla/5.0 (compatible; X11; Linux x86_64; Google-StructuredDataTestingTool; +http://www.google.com/webmasters/tools/richsnippets)'
 GRAPESHOT: 'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)'
+JOBSEEKER: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) JobBot/5.0 (compatible; +http://www.jobseeker.com.au/bot.html) Safari/538.1'
 LINKDEXBOT: 'Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)'
 LOAD_TIME_BOT: 'Mozilla/5.0 (compatible; LoadTimeBot/0.9; +http://www.loadtime.net/bot.html)'
 LTX71: 'ltx71 - (http://ltx71.com/)'
 MAIL_RU: 'Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)'
+MAPPYDATA: 'Mozilla/5.0 (compatible; Mappy/1.0; +http://mappydata.net/bot/)'
 MEGAINDEX_RU: 'Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)'
 MRCHROME: 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.107 Amigo/45.0.2454.107 MRCHROME SOC Safari/537.36'
 MSNBOT: 'msnbot/2.0b (+http://search.msn.com/msnbot.htm)'
@@ -30,21 +40,29 @@ MSNBOT_MEDIA: 'msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)'
 NETCRAFT2: Netcraft SSL Server Survey - contact info@netcraft.com
 NETCRAFT: Mozilla/5.0 (compatible; NetcraftSurveyAgent/1.0; +info@netcraft.com)
 NEWRELICPINGER: NewRelicPinger/1.0 (12345)
+PAESSLER: Mozilla/5.0 (compatible; PRTG Network Monitor (www.paessler.com); Windows)
 PRIVACYAWAREBOT: 'Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)'
 PROXIMIC: 'Mozilla/5.0 (compatible; proximic; +http://www.proximic.com/info/spider.php)'
 QUERYSEEKER: 'QuerySeekerSpider ( http://queryseeker.com/bot.html )'
 SCRAPY: 'Scrapy/0.18.4 (+http://scrapy.org)'
+SEMANTICBOT: 'Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)'
+SEO_AUDIT: 'Mozilla/5.0 (compatible; seo-audit-check-bot/1.0)'
+SEODIVER: 'Mozilla/5.0 (compatible; SEOdiver/1.0; +http://www.seodiver.com/bot)'
 SEOKICKS: 'Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)'
 SISTRIX: 'Mozilla/5.0 (compatible; SISTRIX Crawler; http://crawler.sistrix.net/)'
 SOCIALRANKIO: SocialRankIOBot; http://socialrank.io/about
 SQUIDER: 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110 Safari/537.36 Squider/0.01'
 STRIPE: 'Stripe/1.0 (+https://stripe.com/docs/webhooks)'
 SWIFTYPE: 'Swiftbot'
+TEERAID: 'Mozilla/5.0 (compatible; TeeRaidBot; +https://teeraid.com/bot/)'
 TINEYE: 'TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)'
 TRAACKR: 'Traackr.com'
 WATCHSUMO: 'Mozilla/5.0 (compatible) WatchSumo/1.0.0 (http://www.watchsumo.com)'
+WEBCEO: 'Mozilla/5.0 (compatible; online-webceo-bot/1.0; +http://online.webceo.com)'
 WHATSAPP: 'WhatsApp/2.17.38 Mozilla/5.0 (Linux; U; Android 6.1; en-us; DV Build/Donut) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36'
+WORIOBOT: 'Mozilla/5.0 (compatible; woriobot +http://worio.com)'
 YAHOO_AD_MONITORING: 'Mozilla/5.0 (compatible; Yahoo Ad monitoring; https://help.yahoo.com/kb/yahoo-ad-monitoring-SLN24857.html)'
 YAHOO_SLURP: 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)'
 YANDEX_DIRECT: 'Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)'
 YANDEX_METRIKA: 'Mozilla/5.0 (compatible; YandexMetrika/3.0; +http://yandex.com/bots)'
+YANGA: 'Yanga WorldSearch Bot v1.1/beta (http://www.yanga.co.uk/)'

data/test/unit/accept_language_test.rb

@@ -139,4 +139,12 @@ class AcceptLanguageTest < Minitest::Test
     assert_language result[1], code: "ru", region: nil, quality: 0.9
     assert_language result[2], code: "de", region: nil, quality: 0.8
   end
+
+  test "handles invalid quality values that look like a number" do
+    accept_language = "fr-CH, fr;q=0.9, en;q=0.8, de;q=0..7, *;q=0.5"
+    result = Browser::AcceptLanguage.parse(accept_language)
+
+    assert_equal 5, result.size
+    assert_language result[3], code: "de", region: nil, quality: 0.7
+  end
 end

data/test/unit/bots_test.rb

@@ -27,6 +27,15 @@ class BotsTest < Minitest::Test
     assert browser.bot?
   end
 
+  test "allows setting nil as user agent for bots" do
+    browser = Browser.new(nil)
+    refute browser.bot?
+
+    Browser::Bot.detect_empty_ua!
+    browser = Browser.new(nil)
+    assert browser.bot?
+  end
+
   test "doesn't detect mozilla as a bot when considering empty UA" do
     Browser::Bot.detect_empty_ua!
     browser = Browser.new("Mozilla")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: browser
 version: !ruby/object:Gem::Version
-  version: 2.5.2
+  version: 2.5.3
 platform: ruby
 authors:
 - Nando Vieira
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-26 00:00:00.000000000 Z
+date: 2018-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -172,6 +172,7 @@ files:
 - browser.gemspec
 - config.ru
 - gemfiles/rails4.gemfile
+- gemfiles/rails5.gemfile
 - languages.yml
 - lib/browser.rb
 - lib/browser/accept_language.rb
@@ -315,7 +316,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Do some browser detection with Ruby.