legitbot 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 394b73c86f4a4aefe5cd96a6483ec5f000aaed04
-  data.tar.gz: 7b0fbc498167e635487b4e659dcd88aa95a7360a
+  metadata.gz: f26aea921fa62377a7a180c49ac426cc42168c98
+  data.tar.gz: fb2fa644e6cf65edf0f068417b63d0a650ceac5f
 SHA512:
-  metadata.gz: 82e4a3b94efee99fee6a0cacea416c9596e9d383def2ea5191211ce087f3cf46b5240a87c43d7859a90560eca04c07c3eedebe9b4efb389d9697e28f364f749c
-  data.tar.gz: bd7ff484dc01003c95b6dbd399f4c795480a7d1a194d6c09e98b4d03bc51b26790f6c4b6190ae06c89e0afcc71c50d293b5fb9a8a3a5f078c4d7d87bb70bcc76
+  metadata.gz: 4c909f01468f9a4ea1a8f8cee045d03f1d0736f1868d43172f639379bb05a0b7ba1092ce678465a26495df282fe98b0f0a2542903390b5782b94230a62dff20d
+  data.tar.gz: a8b36a573cda2eaba60cca50cc6a8a28356278257e634b8392e0a3ff719627d73eab9835ae9e3b21a248cd465362c904037c0c0f286533fbd2dabe68ca9532fb

data/README.md

@@ -1,4 +1,4 @@
-# Legitbot
+# Legitbot [![Build Status](https://secure.travis-ci.org/alaz/legitbot.png?branch=master)](http://travis-ci.org/alaz/legitbot) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
 
 Ruby gem to check if an IP really belongs to some bot, typically a search
 engine. This can of much help if one wants to protect his/her web site from
@@ -31,10 +31,21 @@ Rack::Attack.blocklist("fake Googlebot") do |req|
 end
 ```
 
+## Supported
+
+* [Google crawlers](https://support.google.com/webmasters/answer/1061943)
+* [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
+* [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
+* [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
+* [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
+* [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
+
 ## Issues, problems, plans
 
 * Rails middleware
-* Facebook: https://developers.facebook.com/docs/sharing/webmasters/crawler
+* More testing for Facebook
+* Review for thread safety
+* Make it possible to reload Facebook IP ranges
 
 ## License
 

data/Rakefile

@@ -12,4 +12,11 @@ Rake::TestTask.new do |t|
   t.verbose = true
 end
 
+desc 'Start a console'
+task :console do
+  require 'irb'
+  ARGV.clear
+  IRB.start
+end
+
 task default: %w[test]

data/legitbot.gemspec

@@ -15,6 +15,9 @@ Gem::Specification.new do |spec|
     "made by a real search engine, not a fake"
 
   spec.required_ruby_version = '>= 2.0.0'
+  spec.add_dependency "irrc"
+  spec.add_dependency "segment_tree"
+  spec.add_dependency "concurrent-ruby"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
 

data/lib/legitbot/baidu.rb

@@ -3,10 +3,6 @@ module Legitbot
   class Baidu < BotMatch
     ValidDomains = ["baidu.com.", "baidu.jp."]
 
-    def initialize(ip, resolver_config = nil)
-      super(ip, resolver_config)
-    end
-
     def valid?
       subdomain_of?(*Baidu::ValidDomains)
     end

data/lib/legitbot/bing.rb

@@ -3,10 +3,6 @@ module Legitbot
   class Bing < BotMatch
     ValidDomains = ["search.msn.com."]
 
-    def initialize(ip, resolver_config = nil)
-      super(ip, resolver_config)
-    end
-
     def valid?
       subdomain_of?(*Bing::ValidDomains) && reverse_resolves?
     end

data/lib/legitbot/duckduckgo.rb

@@ -3,10 +3,6 @@ module Legitbot
   class DuckDuckGo < BotMatch
     ValidIPs = %w(72.94.249.34 72.94.249.35 72.94.249.36 72.94.249.37 72.94.249.38)
 
-    def initialize(ip, resolver_config = nil)
-      super(ip, resolver_config)
-    end
-
     def valid?
       DuckDuckGo::ValidIPs.include? @ip
     end

data/lib/legitbot/facebook.rb

@@ -0,0 +1,33 @@
+require 'segment_tree'
+require 'irrc'
+require 'concurrent'
+
+module Legitbot
+  # https://developers.facebook.com/docs/sharing/webmasters/crawler
+
+  class Facebook < BotMatch
+    AS = 'AS32934'
+    ValidIPs = Concurrent::Delay.new do
+      client = Irrc::Client.new
+      client.query :radb, 'AS32934'
+      results = client.perform
+
+      Hash[%i(ipv4 ipv6).map { |k|
+        [k, SegmentTree.new(results[AS][k][AS].map { |cidr|
+            [IPAddr.new(cidr).to_range, true]
+          })]
+      }]
+    end
+
+    def valid?
+      ip = IPAddr.new(@ip)
+      if ip.ipv4?
+        ValidIPs.value[:ipv4].find(ip)
+      else
+        ValidIPs.value[:ipv6].find(ip)
+      end
+    end
+  end
+
+  rule Legitbot::Facebook, %w(facebookhit facebookexternalhit)
+end

data/lib/legitbot/google.rb

@@ -7,10 +7,6 @@ module Legitbot
   class Google < BotMatch
     ValidDomains = ["google.com.", "googlebot.com."]
 
-    def initialize(ip, resolver_config = nil)
-      super(ip, resolver_config)
-    end
-
     def valid?
       subdomain_of?(*Google::ValidDomains) && reverse_resolves?
     end

data/lib/legitbot/version.rb

@@ -1,3 +1,3 @@
 module Legitbot
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 end

data/lib/legitbot/yandex.rb

@@ -4,10 +4,6 @@ module Legitbot
   class Yandex < BotMatch
     ValidDomains = ["yandex.ru.", "yandex.net.", "yandex.com."]
 
-    def initialize(ip, resolver_config = nil)
-      super(ip, resolver_config)
-    end
-
     def valid?
       subdomain_of?(*Yandex::ValidDomains) && reverse_resolves?
     end

data/lib/legitbot.rb

@@ -2,8 +2,10 @@ require 'resolv'
 
 require_relative 'legitbot/legitbot'
 require_relative 'legitbot/botmatch'
-require_relative 'legitbot/google'
-require_relative 'legitbot/yandex'
-require_relative 'legitbot/bing'
+
 require_relative 'legitbot/baidu'
+require_relative 'legitbot/bing'
 require_relative 'legitbot/duckduckgo'
+require_relative 'legitbot/facebook'
+require_relative 'legitbot/google'
+require_relative 'legitbot/yandex'

data/test/facebook_test.rb

@@ -0,0 +1,22 @@
+require 'minitest/autorun'
+require 'legitbot'
+
+class FacebookTest < Minitest::Test
+  def test_valid_ip
+    ip = "69.63.186.89"
+    match = Legitbot::Facebook.new(ip)
+    assert match.valid?, msg: "#{ip} is a valid Facebook IP"
+  end
+
+  def test_invalid_ip
+    ip = "127.0.0.1"
+    match = Legitbot::Facebook.new(ip)
+    assert match.fake?, msg: "#{ip} is a fake Facebook IP"
+  end
+
+  def test_user_agent
+    bot = Legitbot.bot("facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)", "31.13.76.56")
+    assert_equal "Facebook", bot.detected_as
+    assert bot.valid?, msg: "A valid Facebook User-agent and IP"
+  end
+end

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2016-12-21 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: irrc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: segment_tree
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -57,11 +99,13 @@ files:
 - lib/legitbot/bing.rb
 - lib/legitbot/botmatch.rb
 - lib/legitbot/duckduckgo.rb
+- lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
 - lib/legitbot/legitbot.rb
 - lib/legitbot/version.rb
 - lib/legitbot/yandex.rb
 - test/botmatch_test.rb
+- test/facebook_test.rb
 - test/google_test.rb
 - test/legitbot_test.rb
 homepage: https://github.com/alaz/legitbot
@@ -91,5 +135,6 @@ specification_version: 4
 summary: Validate Web request was made by legitimate search engine
 test_files:
 - test/botmatch_test.rb
+- test/facebook_test.rb
 - test/google_test.rb
 - test/legitbot_test.rb