legitbot 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f26aea921fa62377a7a180c49ac426cc42168c98
-  data.tar.gz: fb2fa644e6cf65edf0f068417b63d0a650ceac5f
+  metadata.gz: 9624f762d4c29b00c2baf6f53ec6f6dacaec29d4
+  data.tar.gz: e6c2650c5471cb9075e9450fd8a5894f074ee05b
 SHA512:
-  metadata.gz: 4c909f01468f9a4ea1a8f8cee045d03f1d0736f1868d43172f639379bb05a0b7ba1092ce678465a26495df282fe98b0f0a2542903390b5782b94230a62dff20d
-  data.tar.gz: a8b36a573cda2eaba60cca50cc6a8a28356278257e634b8392e0a3ff719627d73eab9835ae9e3b21a248cd465362c904037c0c0f286533fbd2dabe68ca9532fb
+  metadata.gz: dfa1e4d4ec277f2785e49f5af55a630aed08271fa4c7dd24d93d2155d8da8f9522947169a392a2111d6519ab74a0fa3d6410f184f7f67e067403c421ab56ecc7
+  data.tar.gz: f3e765ee7215424adc1a91e990734bc8409634e321a0c2c37dab83e426c438be7134f432cf0424097489d7ce2397fd4a56711da3349f84cbef83ef26aace1f22

data/.gitignore

@@ -2,3 +2,4 @@ Gemfile.lock
 .bundle
 *.gem
 *.gemfile.lock
+/tags

data/README.md

@@ -33,19 +33,21 @@ end
 
 ## Supported
 
-* [Google crawlers](https://support.google.com/webmasters/answer/1061943)
-* [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
-* [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
+* [Applebot](https://support.apple.com/en-us/HT204683)
 * [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
+* [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
 * [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
 * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
+* [Google crawlers](https://support.google.com/webmasters/answer/1061943)
+* [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
+* [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
 
 ## Issues, problems, plans
 
 * Rails middleware
 * More testing for Facebook
-* Review for thread safety
 * Make it possible to reload Facebook IP ranges
+* Bots masquerading as someone else, e.g. `Telegram (like Twitter)` - what to do?
 
 ## License
 

data/legitbot.gemspec

@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.0.0'
   spec.add_dependency "irrc"
   spec.add_dependency "segment_tree"
-  spec.add_dependency "concurrent-ruby"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
 

data/lib/legitbot.rb

@@ -1,11 +1,11 @@
-require 'resolv'
-
 require_relative 'legitbot/legitbot'
 require_relative 'legitbot/botmatch'
 
+require_relative 'legitbot/apple'
 require_relative 'legitbot/baidu'
 require_relative 'legitbot/bing'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
+require_relative 'legitbot/pinterest'
 require_relative 'legitbot/yandex'

data/lib/legitbot/apple.rb

@@ -0,0 +1,16 @@
+require 'ipaddr'
+
+module Legitbot
+  # https://support.apple.com/en-us/HT204683
+
+  class Apple < BotMatch
+    Range = IPAddr.new('17.0.0.0/8')
+
+    def valid?
+      ip = IPAddr.new @ip
+      Range.include? ip
+    end
+  end
+
+  rule Legitbot::Apple, %w(Applebot)
+end

data/lib/legitbot/botmatch.rb

@@ -1,3 +1,6 @@
+require 'resolv'
+require 'ipaddr'
+
 module Legitbot
   ##
   # Represents a bot instance match. Typical methods are

data/lib/legitbot/facebook.rb

@@ -1,13 +1,16 @@
 require 'segment_tree'
 require 'irrc'
-require 'concurrent'
+require 'monitor'
+require 'ipaddr'
 
 module Legitbot
   # https://developers.facebook.com/docs/sharing/webmasters/crawler
 
   class Facebook < BotMatch
+    lock = Monitor.new
+
     AS = 'AS32934'
-    ValidIPs = Concurrent::Delay.new do
+    ValidIPs = lock.synchronize do
       client = Irrc::Client.new
       client.query :radb, 'AS32934'
       results = client.perform
@@ -21,11 +24,7 @@ module Legitbot
 
     def valid?
       ip = IPAddr.new(@ip)
-      if ip.ipv4?
-        ValidIPs.value[:ipv4].find(ip)
-      else
-        ValidIPs.value[:ipv6].find(ip)
-      end
+      ValidIPs[ip.ipv4? ? :ipv4 : :ipv6].find(ip)
     end
   end
 

data/lib/legitbot/google.rb

@@ -1,5 +1,3 @@
-require 'resolv'
-
 module Legitbot
   # https://support.google.com/webmasters/answer/1061943
   # https://support.google.com/webmasters/answer/80553

data/lib/legitbot/pinterest.rb

@@ -0,0 +1,13 @@
+module Legitbot
+  # https://help.pinterest.com/en/articles/about-pinterest-crawler-0
+
+  class Pinterest < BotMatch
+    ValidDomains = ["pinterest.com."]
+
+    def valid?
+      subdomain_of?(*Pinterest::ValidDomains) && reverse_resolves?
+    end
+  end
+
+  rule Legitbot::Pinterest, %w(Pinterestbot Pinterest)
+end

data/lib/legitbot/version.rb

@@ -1,3 +1,3 @@
 module Legitbot
-  VERSION = '0.1.0'
+  VERSION = '0.1.2'
 end

data/test/apple_test.rb

@@ -0,0 +1,22 @@
+require 'minitest/autorun'
+require 'legitbot'
+
+class AppleTest < Minitest::Test
+  def test_valid_ip
+    ip = "17.58.98.60"
+    match = Legitbot::Apple.new(ip)
+    assert match.valid?, msg: "#{ip} is a valid Applebot IP"
+  end
+
+  def test_invalid_ip
+    ip = "127.0.0.1"
+    match = Legitbot::Apple.new(ip)
+    assert match.fake?, msg: "#{ip} is a fake Applebot IP"
+  end
+
+  def test_user_agent
+    bot = Legitbot.bot("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)", "17.58.98.60")
+    assert_equal "Apple", bot.detected_as
+    assert bot.valid?, msg: "A valid Applebot User-agent and IP"
+  end
+end

data/test/botmatch_test.rb

@@ -3,22 +3,22 @@ require 'legitbot'
 
 class BotMatchTest < Minitest::Test
   def test_reverse_name
-    match = Legitbot::BotMatch.new "66.249.78.6"
-    assert_equal "crawl-66-249-78-6.googlebot.com", match.reverse_name
+    match = Legitbot::BotMatch.new "66.249.64.141"
+    assert_equal "crawl-66-249-64-141.googlebot.com", match.reverse_name
   end
 
   def test_reverse_ip
-    match = Legitbot::BotMatch.new "66.249.78.6"
-    assert_equal "66.249.78.6", match.reversed_ip
+    match = Legitbot::BotMatch.new "66.249.64.141"
+    assert_equal "66.249.64.141", match.reversed_ip
   end
 
   def test_reverse_resolves
-    match = Legitbot::BotMatch.new "66.249.78.6"
+    match = Legitbot::BotMatch.new "66.249.64.141"
     assert_equal true, match.reverse_resolves?
   end
 
   def test_valid_class_syntax
-    assert Legitbot::Google.valid?("66.249.78.6"), msg: "Valid Googlebot"
+    assert Legitbot::Google.valid?("66.249.64.141"), msg: "Valid Googlebot"
     assert Legitbot::Google.fake?("149.210.164.47"), msg: "Fake Googlebot"
   end
 end

data/test/google_test.rb

@@ -11,7 +11,7 @@ class GoogleTest < Minitest::Test
   end
 
   def test_valid_ip
-    ip = "66.249.78.6"
+    ip = "66.249.64.141"
     match = Legitbot::Google.new ip
     reverse_name = match.reverse_name
     assert match.subdomain_of?("googlebot.com."), msg: "#{reverse_name} is a subdomain of googlebot.com"
@@ -25,13 +25,13 @@ class GoogleTest < Minitest::Test
   end
 
   def test_valid_ua
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.78.6")
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.64.141")
     assert bot, msg: "Googlebot detected from User-Agent"
     assert bot.valid?, msg: "Valid Googlebot"
   end
 
   def test_engine_name
-    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.78.6")
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.64.141")
     assert_equal "Google", bot.detected_as
   end
 end

data/test/pinterest_test.rb

@@ -0,0 +1,37 @@
+require 'minitest/autorun'
+require 'legitbot'
+
+class PinterestTest < Minitest::Test
+  def test_malicious_ip
+    ip = "149.210.164.47"
+    match = Legitbot::Pinterest.new ip
+    reverse_name = match.reverse_name
+    assert !match.subdomain_of?("pinterest.com."), msg: "#{reverse_name} is not a subdomain of pinterest.com"
+    assert !match.valid?, msg: "#{ip} is not a real Pinterest IP"
+  end
+
+  def test_valid_ip
+    ip = "54.236.1.11"
+    match = Legitbot::Pinterest.new ip
+    reverse_name = match.reverse_name
+    assert match.subdomain_of?("pinterest.com."), msg: "#{reverse_name} is a subdomain of pinterest.com"
+    assert match.valid?, msg: "#{ip} is a valid Pinterest IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "149.210.164.47")
+    assert bot, msg: "Pinterest detected from User-Agent"
+    assert !bot.valid?, msg: "Not a valid Pinterest"
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "54.236.1.11")
+    assert bot, msg: "Pinterest detected from User-Agent"
+    assert bot.valid?, msg: "Valid Pinterest"
+  end
+
+  def test_engine_name
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Pinterestbot/1.0; +https://www.pinterest.com/bot.html)", "54.236.1.11")
+    assert_equal "Pinterest", bot.detected_as
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-21 00:00:00.000000000 Z
+date: 2018-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: irrc
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: concurrent-ruby
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -95,6 +81,7 @@ files:
 - Rakefile
 - legitbot.gemspec
 - lib/legitbot.rb
+- lib/legitbot/apple.rb
 - lib/legitbot/baidu.rb
 - lib/legitbot/bing.rb
 - lib/legitbot/botmatch.rb
@@ -102,12 +89,15 @@ files:
 - lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
 - lib/legitbot/legitbot.rb
+- lib/legitbot/pinterest.rb
 - lib/legitbot/version.rb
 - lib/legitbot/yandex.rb
+- test/apple_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
 - test/google_test.rb
 - test/legitbot_test.rb
+- test/pinterest_test.rb
 homepage: https://github.com/alaz/legitbot
 licenses:
 - Apache-2.0
@@ -129,12 +119,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: Validate Web request was made by legitimate search engine
 test_files:
+- test/legitbot_test.rb
+- test/pinterest_test.rb
+- test/apple_test.rb
+- test/google_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
-- test/google_test.rb
-- test/legitbot_test.rb