legitbot 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 394b73c86f4a4aefe5cd96a6483ec5f000aaed04
+  data.tar.gz: 7b0fbc498167e635487b4e659dcd88aa95a7360a
+SHA512:
+  metadata.gz: 82e4a3b94efee99fee6a0cacea416c9596e9d383def2ea5191211ce087f3cf46b5240a87c43d7859a90560eca04c07c3eedebe9b4efb389d9697e28f364f749c
+  data.tar.gz: bd7ff484dc01003c95b6dbd399f4c795480a7d1a194d6c09e98b4d03bc51b26790f6c4b6190ae06c89e0afcc71c50d293b5fb9a8a3a5f078c4d7d87bb70bcc76

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+.bundle
+*.gem
+*.gemfile.lock

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.2

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (C) 2015 Alexander Azarov <self@alaz.me>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,50 @@
+# Legitbot
+
+Ruby gem to check if an IP really belongs to some bot, typically a search
+engine. This can of much help if one wants to protect his/her web site from
+malicious scanners who pretend to be e.g. a Googlebot.
+
+## Usage
+
+Suppose you have a Web request and you'd like to make sure it's not from a fake
+search engine:
+
+```ruby
+bot = Legitbot.bot(userAgent, ip)
+```
+
+`bot` will be `nil` if no bot signature was found in the `User-Agent`. Otherwise,
+it will be an instance with methods
+
+```ruby
+bot.detected_as # => "Google"
+bot.valid? # => true
+bot.fake? # => false
+```
+
+Sometimes you already know what search engine to expect. For example, you may
+be using [rack-attack](https://github.com/kickstarter/rack-attack):
+
+```ruby
+Rack::Attack.blocklist("fake Googlebot") do |req|
+  req.user_agent =~ %r(Googlebot) && Legitbot::Google.fake?(req.ip)
+end
+```
+
+## Issues, problems, plans
+
+* Rails middleware
+* Facebook: https://developers.facebook.com/docs/sharing/webmasters/crawler
+
+## License
+
+Apache 2.0
+
+## References
+
+* I have initially created Play Framework version in Scala: [play-legitbot](https://github.com/osinka/play-legitbot)
+* Article [When (Fake) Googlebots Attack Your Rails App](http://jessewolgamott.com/blog/2015/11/17/when-fake-googlebots-attack-your-rails-app/)
+* [Voight-Kampff](https://github.com/biola/Voight-Kampff) is a Ruby gem which
+  detects bots by `User-Agent`
+* [browser](https://github.com/fnando/browser) is a Ruby gem which may tell
+  you if the request comes from a search engine.

data/Rakefile

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+  t.warning = true
+  t.verbose = true
+end
+
+task default: %w[test]

data/legitbot.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
+require "legitbot/version"
+
+Gem::Specification.new do |spec|
+  spec.name = 'legitbot'
+  spec.version = Legitbot::VERSION
+  spec.license = 'Apache-2.0'
+
+  spec.author = "Alexander Azarov"
+  spec.email = "self@alaz.me"
+  spec.homepage = "https://github.com/alaz/legitbot"
+  spec.summary = %q{Validate Web request was made by legitimate search engine}
+  spec.description = "A library to make sure a Web request has been "\
+    "made by a real search engine, not a fake"
+
+  spec.required_ruby_version = '>= 2.0.0'
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+
+  spec.files = `git ls-files`.split($/)
+  spec.rdoc_options = ["--charset=UTF-8"]
+  spec.test_files = Dir.glob("test/**/*")
+end

data/lib/legitbot.rb

@@ -0,0 +1,9 @@
+require 'resolv'
+
+require_relative 'legitbot/legitbot'
+require_relative 'legitbot/botmatch'
+require_relative 'legitbot/google'
+require_relative 'legitbot/yandex'
+require_relative 'legitbot/bing'
+require_relative 'legitbot/baidu'
+require_relative 'legitbot/duckduckgo'

data/lib/legitbot/baidu.rb

@@ -0,0 +1,16 @@
+module Legitbot
+  # http://help.baidu.com/question?prod_en=master&class=498&id=1000973
+  class Baidu < BotMatch
+    ValidDomains = ["baidu.com.", "baidu.jp."]
+
+    def initialize(ip, resolver_config = nil)
+      super(ip, resolver_config)
+    end
+
+    def valid?
+      subdomain_of?(*Baidu::ValidDomains)
+    end
+  end
+
+  rule Legitbot::Baidu, %w(Baiduspider)
+end

data/lib/legitbot/bing.rb

@@ -0,0 +1,16 @@
+module Legitbot
+  # https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/
+  class Bing < BotMatch
+    ValidDomains = ["search.msn.com."]
+
+    def initialize(ip, resolver_config = nil)
+      super(ip, resolver_config)
+    end
+
+    def valid?
+      subdomain_of?(*Bing::ValidDomains) && reverse_resolves?
+    end
+  end
+
+  rule Legitbot::Bing, %w(Bingbot bingbot)
+end

data/lib/legitbot/botmatch.rb

@@ -0,0 +1,58 @@
+module Legitbot
+  ##
+  # Represents a bot instance match. Typical methods are
+  # +valid?+, +fake?+ and +detected_as+
+  #
+  class BotMatch
+    def initialize(ip, resolver_config = nil)
+      @dns = Resolv::DNS.new(resolver_config)
+      @ip = ip
+    end
+
+    ##
+    # Returns a Resolv::DNS::Name instance with
+    # the reverse name
+    def reverse_domain
+      @reverse_domain ||= @dns.getname(@ip)
+    end
+
+    ##
+    # Returns a String with the reverse name
+    def reverse_name
+      reverse_domain.to_s
+    end
+
+    ##
+    # Returns a String with IP created from the reverse name
+    def reversed_ip
+      @reverse_ip ||= @dns.getaddress(reverse_name)
+      @reverse_ip.to_s
+    end
+
+    def reverse_resolves?
+      reversed_ip == @ip
+    end
+
+    def subdomain_of?(*domains)
+      domains.any? { |d|
+        reverse_domain.subdomain_of? Resolv::DNS::Name.create(d)
+      }
+    end
+
+    def detected_as
+      self.class.name.split('::').last
+    end
+
+    def fake?
+      !valid?
+    end
+
+    def self.valid?(ip, resolver_config = nil)
+      self.new(ip, resolver_config).valid?
+    end
+
+    def self.fake?(ip, resolver_config = nil)
+      self.new(ip, resolver_config).fake?
+    end
+  end
+end

data/lib/legitbot/duckduckgo.rb

@@ -0,0 +1,16 @@
+module Legitbot
+  # https://duckduckgo.com/duckduckbot
+  class DuckDuckGo < BotMatch
+    ValidIPs = %w(72.94.249.34 72.94.249.35 72.94.249.36 72.94.249.37 72.94.249.38)
+
+    def initialize(ip, resolver_config = nil)
+      super(ip, resolver_config)
+    end
+
+    def valid?
+      DuckDuckGo::ValidIPs.include? @ip
+    end
+  end
+
+  rule Legitbot::DuckDuckGo, %w(DuckDuckGo)
+end

data/lib/legitbot/google.rb

@@ -0,0 +1,20 @@
+require 'resolv'
+
+module Legitbot
+  # https://support.google.com/webmasters/answer/1061943
+  # https://support.google.com/webmasters/answer/80553
+
+  class Google < BotMatch
+    ValidDomains = ["google.com.", "googlebot.com."]
+
+    def initialize(ip, resolver_config = nil)
+      super(ip, resolver_config)
+    end
+
+    def valid?
+      subdomain_of?(*Google::ValidDomains) && reverse_resolves?
+    end
+  end
+
+  rule Legitbot::Google, %w(Googlebot Mediapartners-Google AdsBot-Google)
+end

data/lib/legitbot/legitbot.rb

@@ -0,0 +1,31 @@
+module Legitbot
+  @rules = []
+
+  ##
+  # Lookup a bot based on its signature from +User-Agent+ header.
+  #
+  # If a block given, passes the found bot to the block.
+  #
+  # Returns +nil+ if no bot found and a bot match instance
+  # otherwise.
+  # :yields: a found bot
+  #
+  def self.bot(userAgent, ip, resolver_config = nil)
+    bot =
+      @rules.select { |rule|
+        rule[:fragments].any? {|f| userAgent.index f}
+      }.map { |rule|
+        rule[:class].new(ip, resolver_config)
+      }.first
+
+    if bot && block_given?
+      yield bot
+    else
+      bot
+    end
+  end
+
+  def self.rule(clazz, fragments)
+    @rules << {:class => clazz, :fragments => fragments}
+  end
+end

data/lib/legitbot/version.rb

@@ -0,0 +1,3 @@
+module Legitbot
+  VERSION = '0.0.1'
+end

data/lib/legitbot/yandex.rb

@@ -0,0 +1,20 @@
+module Legitbot
+  # https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml
+
+  class Yandex < BotMatch
+    ValidDomains = ["yandex.ru.", "yandex.net.", "yandex.com."]
+
+    def initialize(ip, resolver_config = nil)
+      super(ip, resolver_config)
+    end
+
+    def valid?
+      subdomain_of?(*Yandex::ValidDomains) && reverse_resolves?
+    end
+  end
+
+  rule Legitbot::Yandex, %w(YandexBot YandexMobileBot YandexImages YandexVideo
+    YandexMedia YandexBlogs YandexFavicons YandexWebmaster YandexPagechecker
+    YandexImageResizer YandexSitelinks YandexMetrika YandexDirectDyn YandexRCA
+    YaDirectFetcher YandexAntivirus YandexVertis YandexCalendar)
+end

data/test/botmatch_test.rb

@@ -0,0 +1,24 @@
+require 'minitest/autorun'
+require 'legitbot'
+
+class BotMatchTest < Minitest::Test
+  def test_reverse_name
+    match = Legitbot::BotMatch.new "66.249.78.6"
+    assert_equal "crawl-66-249-78-6.googlebot.com", match.reverse_name
+  end
+
+  def test_reverse_ip
+    match = Legitbot::BotMatch.new "66.249.78.6"
+    assert_equal "66.249.78.6", match.reversed_ip
+  end
+
+  def test_reverse_resolves
+    match = Legitbot::BotMatch.new "66.249.78.6"
+    assert_equal true, match.reverse_resolves?
+  end
+
+  def test_valid_class_syntax
+    assert Legitbot::Google.valid?("66.249.78.6"), msg: "Valid Googlebot"
+    assert Legitbot::Google.fake?("149.210.164.47"), msg: "Fake Googlebot"
+  end
+end

data/test/google_test.rb

@@ -0,0 +1,37 @@
+require 'minitest/autorun'
+require 'legitbot'
+
+class GoogleTest < Minitest::Test
+  def test_malicious_ip
+    ip = "149.210.164.47"
+    match = Legitbot::Google.new ip
+    reverse_name = match.reverse_name
+    assert !match.subdomain_of?("googlebot.com."), msg: "#{reverse_name} is not a subdomain of googlebot.com"
+    assert !match.valid?, msg: "#{ip} is not a real Googlebot IP"
+  end
+
+  def test_valid_ip
+    ip = "66.249.78.6"
+    match = Legitbot::Google.new ip
+    reverse_name = match.reverse_name
+    assert match.subdomain_of?("googlebot.com."), msg: "#{reverse_name} is a subdomain of googlebot.com"
+    assert match.valid?, msg: "#{ip} is a valid Googlebot IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "149.210.164.47")
+    assert bot, msg: "Googlebot detected from User-Agent"
+    assert !bot.valid?, msg: "Not a valid Googlebot"
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.78.6")
+    assert bot, msg: "Googlebot detected from User-Agent"
+    assert bot.valid?, msg: "Valid Googlebot"
+  end
+
+  def test_engine_name
+    bot = Legitbot.bot("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "66.249.78.6")
+    assert_equal "Google", bot.detected_as
+  end
+end

data/test/legitbot_test.rb

@@ -0,0 +1,12 @@
+require 'minitest/autorun'
+require 'legitbot'
+
+class LegitbotTest < Minitest::Test
+  def test_rules
+    assert !Legitbot.bot("Firefox", "127.0.0.1"), msg: "Not a bot"
+
+    Legitbot.bot("Firefox", "127.0.0.1") do |bot|
+      flunk "No bot Firefox is possible"
+    end
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: legitbot
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Alexander Azarov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A library to make sure a Web request has been made by a real search engine,
+  not a fake
+email: self@alaz.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- legitbot.gemspec
+- lib/legitbot.rb
+- lib/legitbot/baidu.rb
+- lib/legitbot/bing.rb
+- lib/legitbot/botmatch.rb
+- lib/legitbot/duckduckgo.rb
+- lib/legitbot/google.rb
+- lib/legitbot/legitbot.rb
+- lib/legitbot/version.rb
+- lib/legitbot/yandex.rb
+- test/botmatch_test.rb
+- test/google_test.rb
+- test/legitbot_test.rb
+homepage: https://github.com/alaz/legitbot
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--charset=UTF-8"
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Validate Web request was made by legitimate search engine
+test_files:
+- test/botmatch_test.rb
+- test/google_test.rb
+- test/legitbot_test.rb