legitbot 0.3.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d775b4e4e434615989b8ef35aca9ac6498b06742ca6e8534f9bb4d6f3d5c9b04
-  data.tar.gz: 239069018d1b043ec99bce8496b3760fc189611f6aa83545835d5f71ce7f94c5
+  metadata.gz: 34d1432c7c405d783b22a46851db88ccdea9f303defeccdd1cf98604bbb6ce09
+  data.tar.gz: a66b586f4b2dca67fb875ea37add6e7d89a7ce5d0705c3d1898d96ecf091036e
 SHA512:
-  metadata.gz: '08e08f94cbb4979566982d91a525f7d03618c9c10e85613bcebfd51f8f8f8b0dab669ba9cd1b67c7a6d6a36c23d23ad88b64996bf252bd884082045ae590d38c'
-  data.tar.gz: 97c4dd80bf62205ea4a03a0bc7570d57520611b4015f3bf9f66b450d78878a3fde1c9097e0ac79691e40d5f29e9ba676aaa3af70f71650adb61c7f4a0ee1ddf0
+  metadata.gz: cad1db2571e939020f74e871365c4748dd78ff18eb8ad9f005ea5bf5b0707835e296afa2601fc6309c994f69b7903d21da788fc219f5d712ee75e1ae9885fb7b
+  data.tar.gz: c72af598d60c55aff35a1b5e244dcde160a67589c588d68b9482d5e5c5f0590441c92505ec94beb462ff70a730ec9aabe80877a2d9db2f72c566c3a9c0b19059

data/.github/workflows/build.yml

@@ -0,0 +1,60 @@
+name: build
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ jruby, 2.6 ]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Cache dependencies
+      uses: actions/cache@v1
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ruby }}-gems-
+    - name: Install dependencies
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run tests
+      run: bundle exec rake test
+
+  lint:
+    needs: test
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby: [ 2.6 ]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Cache dependencies
+      uses: actions/cache@v1
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ruby }}-gems-
+    - name: Install dependencies
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run linter
+      run: bundle exec rubocop

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 *.gemfile.lock
 /pkg
 /tags
+/vendor

data/.rubocop.yml

@@ -0,0 +1,2 @@
+AllCops:
+  CacheRootDirectory: 'vendor'

data/.ruby-version

@@ -0,0 +1 @@
+2.4

data/Gemfile

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 gemspec

data/README.md

@@ -1,4 +1,4 @@
-# Legitbot [![Build Status](https://secure.travis-ci.org/alaz/legitbot.png?branch=master)](http://travis-ci.org/alaz/legitbot) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
+# Legitbot ![](https://github.com/alaz/legitbot/workflows/build/badge.svg) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
 
 Ruby gem to check that an IP belongs to a bot, typically a search
 engine. This can be of help in protecting a web site from fake search
@@ -50,7 +50,9 @@ end
 * [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
 * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
 * [Google crawlers](https://support.google.com/webmasters/answer/1061943)
+* [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
+* [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
 * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
 
 ## License

data/Rakefile

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 require 'rubygems'
 require 'bundler'
 require 'bump/tasks'
-require "rake/testtask"
+require 'rake/testtask'
 Bundler::GemHelper.install_tasks
 
 Bump.tag_by_default = true
 
 Rake::TestTask.new do |t|
-  t.libs << "test"
-  t.test_files = FileList['test/*_test.rb']
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
   t.warning = true
   t.verbose = true
 end

data/legitbot.gemspec

@@ -1,27 +1,28 @@
-# encoding: utf-8
-$LOAD_PATH.push File.expand_path("../lib", __FILE__)
-require "legitbot/version"
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+require 'legitbot/version'
 
 Gem::Specification.new do |spec|
   spec.name = 'legitbot'
   spec.version = Legitbot::VERSION
   spec.license = 'Apache-2.0'
 
-  spec.author = "Alexander Azarov"
-  spec.email = "self@alaz.me"
-  spec.homepage = "https://github.com/alaz/legitbot"
-  spec.summary = %q{Validate requests from Web crawlers: impersonating or not?}
-  spec.description = "A library to make sure a Web request has been "\
-    "made by a real search engine, not a malicious agent"
+  spec.author = 'Alexander Azarov'
+  spec.email = 'self@alaz.me'
+  spec.homepage = 'https://github.com/alaz/legitbot'
+  spec.summary = 'Validate requests from Web crawlers: impersonating or not?'
+  spec.description = 'Does Web request come from a real search engine or from an impersonating agent?'
 
-  spec.required_ruby_version = '>= 2.3.0'
-  spec.add_dependency "irrc", ">= 0.2.1"
-  spec.add_dependency "augmented_interval_tree", ">= 0.1.1"
-  spec.add_development_dependency "bump"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
+  spec.required_ruby_version = '>= 2.4.0'
+  spec.add_dependency 'augmented_interval_tree', '~> 0.1', '>= 0.1.1'
+  spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
+  spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
+  spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
+  spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.74', '>= 0.74.0'
 
-  spec.files = `git ls-files`.split($/)
-  spec.rdoc_options = ["--charset=UTF-8"]
-  spec.test_files = Dir.glob("test/**/*")
+  spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.rdoc_options = ['--charset=UTF-8']
+  spec.test_files = Dir.glob('test/**/*')
 end

data/lib/legitbot.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'legitbot/legitbot'
 require_relative 'legitbot/botmatch'
 
@@ -8,5 +10,7 @@ require_relative 'legitbot/bing'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
+require_relative 'legitbot/oracle'
 require_relative 'legitbot/pinterest'
+require_relative 'legitbot/twitter'
 require_relative 'legitbot/yandex'

data/lib/legitbot/ahrefs.rb

@@ -1,13 +1,18 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://ahrefs.com/robot
   class Ahrefs < BotMatch
-    Ranges = %w(54.36.148.0/24 54.36.149.0/24 54.36.150.0/24 195.154.122.0/24 195.154.123.0/24 195.154.126.0/24 195.154.127.0/24)
-
-    def valid?
-      ip = IPAddr.new @ip
-      Ranges.any? { |range| IPAddr.new(range).include? ip }
-    end
+    ip_ranges %w[
+      54.36.148.0/24
+      54.36.149.0/24
+      54.36.150.0/24
+      195.154.122.0/24
+      195.154.123.0/24
+      195.154.126.0/24
+      195.154.127.0/24
+    ]
   end
 
-  rule Legitbot::Ahrefs, %w(AhrefsBot)
+  rule Legitbot::Ahrefs, %w[AhrefsBot]
 end

data/lib/legitbot/apple.rb

@@ -1,20 +1,20 @@
+# frozen_string_literal: true
+
 require 'ipaddr'
 
-module Legitbot
+module Legitbot # :nodoc:
   # https://support.apple.com/en-us/HT204683
-
   class Apple < BotMatch
-    Range = IPAddr.new('17.0.0.0/8')
-
-    def valid?
-      ip = IPAddr.new @ip
-      Range.include? ip
-    end
+    ip_ranges '17.0.0.0/8'
   end
 
-  class Apple_as_Google < Apple
+  # https://support.apple.com/en-us/HT204683
+  # rubocop:disable Naming/ClassAndModuleCamelCase
+  class Apple_as_Google < BotMatch
+    ip_ranges '17.0.0.0/8'
   end
+  # rubocop:enable Naming/ClassAndModuleCamelCase
 
-  rule Legitbot::Apple, %w(Applebot)
-  rule Legitbot::Apple_as_Google, %w(Googlebot)
+  rule Legitbot::Apple, %w[Applebot]
+  rule Legitbot::Apple_as_Google, %w[Googlebot]
 end

data/lib/legitbot/baidu.rb

@@ -1,12 +1,10 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # http://help.baidu.com/question?prod_en=master&class=498&id=1000973
   class Baidu < BotMatch
-    ValidDomains = ["baidu.com.", "baidu.jp."]
-
-    def valid?
-      subdomain_of?(*Baidu::ValidDomains)
-    end
+    domains 'baidu.com.', 'baidu.jp.', reverse: false
   end
 
-  rule Legitbot::Baidu, %w(Baiduspider)
+  rule Legitbot::Baidu, %w[Baiduspider]
 end

data/lib/legitbot/bing.rb

@@ -1,12 +1,10 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/
   class Bing < BotMatch
-    ValidDomains = ["search.msn.com."]
-
-    def valid?
-      subdomain_of?(*Bing::ValidDomains) && reverse_resolves?
-    end
+    domains 'search.msn.com.'
   end
 
-  rule Legitbot::Bing, %w(Bingbot bingbot)
+  rule Legitbot::Bing, %w[Bingbot bingbot]
 end

data/lib/legitbot/botmatch.rb

@@ -1,5 +1,8 @@
-require 'resolv'
-require 'ipaddr'
+# frozen_string_literal: true
+
+require_relative 'config/resolver'
+require_relative 'validators/domains'
+require_relative 'validators/ip_ranges'
 
 module Legitbot
   ##
@@ -7,61 +10,31 @@ module Legitbot
   # +valid?+, +fake?+ and +detected_as+
   #
   class BotMatch
-    def initialize(ip, resolver_config = nil)
-      @dns = Resolv::DNS.new(resolver_config)
-      @ip = ip
-    end
-
-    ##
-    # Returns a Resolv::DNS::Name instance with
-    # the reverse name
-    def reverse_domain
-      @reverse_domain ||= @dns.getname(@ip)
-    rescue Resolv::ResolvError
-      @reverse_domain ||= nil
-    end
-
-    ##
-    # Returns a String with the reverse name
-    def reverse_name
-      reverse_domain&.to_s
-    end
-
-    ##
-    # Returns a String with IP created from the reverse name
-    def reversed_ip
-      return nil if reverse_name.nil?
+    include Legitbot::Validators::IpRanges
+    include Legitbot::Validators::Domains
 
-      @reverse_ip ||= @dns.getaddress(reverse_name)
-      @reverse_ip.to_s
-    end
-
-    def reverse_resolves?
-      @ip == reversed_ip
-    end
-
-    def subdomain_of?(*domains)
-      return false if reverse_name.nil?
-
-      domains.any? { |d|
-        reverse_domain.subdomain_of? Resolv::DNS::Name.create(d)
-      }
+    def initialize(ip)
+      @ip = ip
     end
 
     def detected_as
       self.class.name.split('::').last.downcase.to_sym
     end
 
+    def valid?
+      valid_ip? && valid_domain?
+    end
+
     def fake?
       !valid?
     end
 
-    def self.valid?(ip, resolver_config = nil)
-      self.new(ip, resolver_config).valid?
+    def self.valid?(ip)
+      new(ip).valid?
     end
 
-    def self.fake?(ip, resolver_config = nil)
-      self.new(ip, resolver_config).fake?
+    def self.fake?(ip)
+      new(ip).fake?
     end
   end
 end

data/lib/legitbot/config/resolver.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'resolv'
+
+module Legitbot
+  module Config
+    module Resolver # :nodoc:
+      def resolver_config(options = nil)
+        @resolver_config = options
+      end
+
+      def resolver
+        @resolver_config ||= Legitbot.resolver_config
+        @resolver ||= Resolv::DNS.new @resolver_config
+      end
+    end
+  end
+end

data/lib/legitbot/duckduckgo.rb

@@ -1,12 +1,23 @@
-module Legitbot
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
-    ValidIPs = %w(50.16.241.113 50.16.241.114 50.16.241.117 50.16.247.234 52.204.97.54 52.5.190.19 54.197.234.188 54.208.100.253 23.21.227.69)
-
-    def valid?
-      DuckDuckGo::ValidIPs.include? @ip
-    end
+    ip_ranges %w[
+      23.21.227.69
+      40.88.21.235
+      50.16.241.113
+      50.16.241.114
+      50.16.241.117
+      50.16.247.234
+      52.204.97.54
+      52.5.190.19
+      54.197.234.188
+      54.208.100.253
+      54.208.102.37
+      107.21.1.8
+    ]
   end
 
-  rule Legitbot::DuckDuckGo, %w(DuckDuckBot)
+  rule Legitbot::DuckDuckGo, %w[DuckDuckBot]
 end

data/lib/legitbot/facebook.rb

@@ -1,48 +1,22 @@
-require 'ipaddr'
+# frozen_string_literal: true
+
 require 'irrc'
-require 'interval_tree'
 
-module Legitbot
+module Legitbot # :nodoc:
   # https://developers.facebook.com/docs/sharing/webmasters/crawler
-
   class Facebook < BotMatch
     AS = 'AS32934'
 
-    def valid?
-      ip = IPAddr.new(@ip)
-      Facebook.valid_ips[ip.ipv4? ? :ipv4 : :ipv6].search(ip.to_i).size > 0
-    end
-
-    @mutex = Mutex.new
-
-    def self.valid_ips
-      @mutex.synchronize { @ips ||= load_ips }
-    end
-
-    def self.reload!
-      @mutex.synchronize { @ips = load_ips }
-    end
-
-    def self.load_ips
-      whois.map do |(family, records)|
-        ranges = records.map do |cidr|
-          range = IPAddr.new(cidr).to_range
-          (range.begin.to_i..range.end.to_i)
-        end
-        [family, IntervalTree::Tree.new(ranges)]
-      end.to_h
-    end
-
-    def self.whois
+    ip_ranges do
       client = Irrc::Client.new
       client.query :radb, AS
       results = client.perform
 
-      %i(ipv4 ipv6).map do |family|
-        [family, results[AS][family][AS]]
-      end.to_h
+      %i[ipv4 ipv6].map do |family|
+        results[AS][family][AS]
+      end.flatten
     end
   end
 
-  rule Legitbot::Facebook, %w(Facebot facebookexternalhit/1.1)
+  rule Legitbot::Facebook, %w[Facebot facebookexternalhit/1.1]
 end