bot_detection 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c246237ec3d560688b788f3484a98e1f0be542e7
-  data.tar.gz: 2602e39c633e6288bc8c8520d129d18dbe3ce589
+  metadata.gz: 3f9cf46bdce8f49486f6a1bc88f966f2068920aa
+  data.tar.gz: 7ec14e78a5cf23c8534642f945186b95ccb8f042
 SHA512:
-  metadata.gz: 44b76dff0c36b98cca4d73f3797ebe1397c1e3b164db6b8b764039296ee627fc95807555262e107cbfab04fa8465d660c9d0f84c667547e5463588ae674a9a2b
-  data.tar.gz: 6c4cedabfe59efd419e517f8d81be89d5f4e011182b5c84e5d66df7d61f24cd41c81b92f9c609ef649e0b1c99995bafecd719d7d79eacc958dca3c5973a180d0
+  metadata.gz: e455e84029af1e41be86f87b381f028ee36443dcb7cb202cceaff4be8516bcdcd50029676d10af21513e4e151f6fdd60f8d19263742567f773cc39f224b72b17
+  data.tar.gz: 964f6ef6ce7fe47fe5b97cdc474462557056c394e79041f6983056ab9589a8be3160242694b0d602313b33c8ed18cf1dbf3e520c1d4d5e4d8bb6f3117823433c

data/.gitignore

@@ -20,3 +20,4 @@ tmp
 *.o
 *.a
 mkmf.log
+.rvmrc

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2
+
+notifications:
+  email:
+    on_success: change
+    on_failure: always

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+#### Release 1.0.7
+
+- added several new user agents, which have been spotted around, to the list
+- added rspec to validate builds on travis
+
 #### Release 1.0.6
 
 - added several new user agents, which have been spotted around, to the list

data/Rakefile

@@ -1,2 +1,4 @@
 require "bundler/gem_tasks"
-
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/bot_detection.gemspec

@@ -5,6 +5,7 @@ require 'bot_detection/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "bot_detection"
+  spec.description   = "Checks a user agent for a web crawler"
   spec.version       = BotDetection::VERSION
   spec.authors       = ["Nils Berenbold"]
   spec.email         = ["nils.berenbold@gmail.com"]
@@ -16,8 +17,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "public_suffix", "~> 1.4.6"
+  spec.add_runtime_dependency "public_suffix", "~> 1.4"
 
+  spec.add_development_dependency "rspec", "~> 3.1"
   spec.add_development_dependency "bundler", "~> 1.6"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rake", "~> 10.4"
 end

data/lib/bot_detection/bot_user_agents.rb

@@ -12,6 +12,7 @@ BotDetection::BotUserAgents = [
   "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11",
   "ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)",
   "Mozilla/5.0 (compatible; Exabot/3.0 (BiggerBetter); +http://www.exabot.com/go/robot)",
+  "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)",
   "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)",
   "Mozilla/5.0 (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)",
   "Twitterbot/1.0",
@@ -41,8 +42,48 @@ BotDetection::BotUserAgents = [
   "bitlybot",
   "Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0",
   "Mozilla/5.0 (Linux; U; Android 4.2.2; bg-bg; CUBOT X6 Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
+  "Mozilla/5.0 (Linux; U; Android 4.2.2; de-de; CUBOT S108 Build/CUBOT_S108) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30",
   "DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)",
   "AppEngine-Google; (+http://code.google.com/appengine; appid: asburyisgay)",
   "Mozilla/5.0 (compatible; SemrushBot/0.98~bl; +http://www.semrush.com/bot.html)",
   "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)",
+  "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
+  "MaxPointCrawler/Nutch-1.1 (maxpoint.crawler at maxpointinteractive dot com)",
+  "woobot/2.0",
+  "techmixx Spider v1 (http://techmixx.com/)",
+  "WordPress/4.0.1; http://novarabota.org",
+  "yacybot (/global; amd64 Windows 7 6.1; java 1.8.0_25; Europe/en) http://yacy.net/bot.html",
+  "Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +http://go.mail.ru/help/robots)",
+  "QuerySeekerSpider ( http://queryseeker.com/bot.html )",
+  "Mozilla/5.0 (compatible; LinkpadBot/1.06; +http://www.linkpad.ru)",
+  "Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)",
+  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-wherecat@moz.com)",
+  "Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)",
+  "Superarama.com - BOT/v.0.1",
+  "Mozilla/5.0 (compatible; memoryBot/1.21.14 +http://mignify.com/bot.html)",
+  "Elmer, the Thinglink ImageBot (http://www.thinglink.com/help/ThinglinkImageBot)",
+  "Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)",
+  "Mozilla/5.0 (compatible; MJ12bot/v1.4.4; http://www.majestic12.co.uk/bot.php?+)",
+  "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; CrystalSemanticsBot http://www.crystalsemantics.com/service-navigation/imprint/useragent/)",
+  "Mozilla/5.0 (compatible;acapbot/0.1;treat like Googlebot)",
+  "Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot)",
+  "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
+  "Wotbox/2.01 (+http://www.wotbox.com/bot/)",
+  "Wotbox/2.0 (bot@wotbox.com; http://www.wotbox.com)",
+  "Mozilla/5.0 (compatible; EveryoneSocialBot/1.0; support@everyonesocial.com http://everyonesocial.com/)",
+  "Jabse.com/2.0 (+http://www.jabse.com/bot.php)",
+  "Cliqzbot/0.1 (+http://cliqz.com/company/cliqzbot)",
+  "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)",
+  "CCBot/2.0 (http://commoncrawl.org/faq/)",
+  "Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)",
+  "Mozilla/5.0 (compatible; uMBot-FC/1.0; mailto: crawling@ubermetrics-technologies.com)",
+  "Mozilla/5.0 (compatible; memoryBot/1.20.311 +http://internetmemory.org/en/)",
+  "Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)",
+  "Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)",
+  "German Wikipedia Broken Weblinks Bot; contact: gifti@tools.wmflabs.org",
+  "Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)",
+  "Mozilla/5.0 (compatible; TwitterCrawler)",
+  "rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)",
+  "crawler4j (http://code.google.com/p/crawler4j/)",
+  "Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)",
 ]

data/lib/bot_detection/search_engine_user_agents.rb

@@ -3,6 +3,7 @@ BotDetection::SearchEngineUserAgents = [
   "Baiduspider+(+http://www.baidu.com/search/spider.htm)",
   "Baiduspider+(+http://www.baidu.com/search/spider_jp.html)",
   "BaiDuSpider",
+  "Mozilla/5.0+(compatible;+Baiduspider-cpro;++http://www.baidu.com/search/spider.html)",
   "DuckDuckBot/1.0; (+http://duckduckgo.com/duckduckbot.html)",
   "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
   "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)",
@@ -28,10 +29,12 @@ BotDetection::SearchEngineUserAgents = [
   "Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)",
   "msnbot/2.1",
   "msnbot/2.0b",
+  "msnbot/2.0b (+http://search.msn.com/msnbot.htm)",
   "msnbot/1.1 (+http://search.msn.com/msnbot.htm)",
   "msnbot/1.1",
   "msnbot/1.0 (+http://search.msn.com/msnbot.htm)",
   "msnbot/0.9 (+http://search.msn.com/msnbot.htm)",
   "msnbot/0.11 ( http://search.msn.com/msnbot.htm)",
   "MSNBOT/0.1 (http://search.msn.com/msnbot.htm)",
+  "Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)",
 ]

data/lib/bot_detection/version.rb

@@ -1,3 +1,3 @@
 module BotDetection
-  VERSION = "1.0.6"
+  VERSION = "1.0.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bot_detection
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: ruby
 authors:
 - Nils Berenbold
@@ -14,52 +14,67 @@ dependencies:
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.6
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.6
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: 
+        version: '10.4'
+description: Checks a user agent for a web crawler
 email:
 - nils.berenbold@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -82,17 +97,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Detects Search Engine crawlers by reverse DNS lookups.