legitbot 0.4.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfc1b3322ff4f85957dabf6790d535f27f99feed47bdb4ff1bba65f6242d31a2
-  data.tar.gz: 32e842cc3d297b3afda0ef9b265121a11c363857a726819f886b441e6c53a53c
+  metadata.gz: 1596ed8c3809fc3c3068f14d8bbbcf84232286fd7157fe724ba9515d195259cd
+  data.tar.gz: f0078d0404d752550adeeaa9c64954cbcec57fef3d0b37c98bc50d765c29bc1f
 SHA512:
-  metadata.gz: 3654c256da13b37045425457a96ac9a8b41c5ae5c0cce49b7898170e2d23a66a5cb7e612503dc1d88dc2e1240dcb07c9ccc5d5aa8439f280144abe969dc0ae7b
-  data.tar.gz: 554e120d1001a71f455aedcd4d30397b22130279a6f99f8e022d40ade50427590dd52e982820399852ec52fbb2c96b7ea1461f82d8e88cb89acc053097136b32
+  metadata.gz: 5640ae1e351bbd697325cd0bed0bdac45c3726e7b31923b7b5c8f51859ed025d4eb368d937a2b3bf9ef6c150d83418f8e019304dec3bc1b010572a4e1598c661
+  data.tar.gz: bf326da52d3adf1b2cfa3693b51cc763daef54c9b175cfb91b437b213722a0f578565db553f1240b6d6ee47d159bb418fdcc1db1f95cb14696b2bfb52e50b75d

data/.github/workflows/build.yml

@@ -0,0 +1,60 @@
+name: build
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ jruby, 2.6 ]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Cache dependencies
+      uses: actions/cache@v1
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ruby }}-gems-
+    - name: Install dependencies
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run tests
+      run: bundle exec rake test
+
+  lint:
+    needs: test
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby: [ 2.6 ]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Cache dependencies
+      uses: actions/cache@v1
+      with:
+        path: vendor/bundle
+        key: ${{ runner.os }}-${{ matrix.ruby }}-gems-${{ hashFiles('**/Gemfile.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.ruby }}-gems-
+    - name: Install dependencies
+      run: |
+        bundle config path vendor/bundle
+        bundle install --jobs 4 --retry 3
+    - name: Run linter
+      run: bundle exec rubocop

data/.gitignore

@@ -4,3 +4,4 @@ Gemfile.lock
 *.gemfile.lock
 /pkg
 /tags
+/vendor

data/.rubocop.yml

@@ -1,8 +1,2 @@
 AllCops:
-  Include:
-    - '**/Gemfile'
-    - '**/Rakefile'
-    - 'lib/**/*.rb'
-    - 'test/**/*.rb'
-  Exclude:
-    - 'pkg/**'
+  CacheRootDirectory: 'vendor'

data/.ruby-version

@@ -0,0 +1 @@
+2.4

data/README.md

@@ -1,4 +1,4 @@
-# Legitbot [![Build Status](https://secure.travis-ci.org/alaz/legitbot.png?branch=master)](http://travis-ci.org/alaz/legitbot) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
+# Legitbot ![](https://github.com/alaz/legitbot/workflows/build/badge.svg) [![Gem Version](https://badge.fury.io/rb/legitbot.svg)](https://badge.fury.io/rb/legitbot)
 
 Ruby gem to check that an IP belongs to a bot, typically a search
 engine. This can be of help in protecting a web site from fake search
@@ -50,7 +50,9 @@ end
 * [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
 * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
 * [Google crawlers](https://support.google.com/webmasters/answer/1061943)
+* [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
+* [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
 * [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
 
 ## License
@@ -65,3 +67,8 @@ Apache 2.0
   detects bots by `User-Agent`
 * [crawler_detect](https://github.com/loadkpi/crawler_detect) is a Ruby gem and Rack
   middleware to detect crawlers by few different request headers, including `User-Agent`
+* Project Honeypot's
+  [http:BL](https://www.projecthoneypot.org/httpbl_api.php) can not only
+  classify IP as a search engine, but also label them as suspicious and
+  reports the number of days since the last activity. My implementation of
+  the protocol in Scala is [here](https://github.com/osinka/httpbl).

data/legitbot.gemspec

@@ -1,28 +1,28 @@
-# encoding: utf-8
-$LOAD_PATH.push File.expand_path("../lib", __FILE__)
-require "legitbot/version"
+# frozen_string_literal: true
+
+$LOAD_PATH.push File.expand_path('lib', __dir__)
+require 'legitbot/version'
 
 Gem::Specification.new do |spec|
   spec.name = 'legitbot'
   spec.version = Legitbot::VERSION
   spec.license = 'Apache-2.0'
 
-  spec.author = "Alexander Azarov"
-  spec.email = "self@alaz.me"
-  spec.homepage = "https://github.com/alaz/legitbot"
-  spec.summary = %q{Validate requests from Web crawlers: impersonating or not?}
-  spec.description = "A library to make sure a Web request has been "\
-    "made by a real search engine, not a malicious agent"
+  spec.author = 'Alexander Azarov'
+  spec.email = 'self@alaz.me'
+  spec.homepage = 'https://github.com/alaz/legitbot'
+  spec.summary = 'Validate requests from Web crawlers: impersonating or not?'
+  spec.description = 'Does Web request come from a real search engine or from an impersonating agent?'
 
-  spec.required_ruby_version = '>= 2.3.0'
-  spec.add_dependency "irrc", ">= 0.2.1"
-  spec.add_dependency "augmented_interval_tree", ">= 0.1.1"
-  spec.add_development_dependency "bump", '>= 0.8.0'
-  spec.add_development_dependency "rake", '>= 12.3.0'
-  spec.add_development_dependency "rubocop", '>= 0.74.0'
-  spec.add_development_dependency "minitest", '>= 5.1.0'
+  spec.required_ruby_version = '>= 2.4.0'
+  spec.add_dependency 'augmented_interval_tree', '~> 0.1', '>= 0.1.1'
+  spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
+  spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
+  spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
+  spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.74', '>= 0.74.0'
 
-  spec.files = `git ls-files`.split($/)
-  spec.rdoc_options = ["--charset=UTF-8"]
-  spec.test_files = Dir.glob("test/**/*")
+  spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.rdoc_options = ['--charset=UTF-8']
+  spec.test_files = Dir.glob('test/**/*')
 end

data/lib/legitbot.rb

@@ -10,5 +10,7 @@ require_relative 'legitbot/bing'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
+require_relative 'legitbot/oracle'
 require_relative 'legitbot/pinterest'
+require_relative 'legitbot/twitter'
 require_relative 'legitbot/yandex'

data/lib/legitbot/duckduckgo.rb

@@ -4,6 +4,8 @@ module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
     ip_ranges %w[
+      23.21.227.69
+      40.88.21.235
       50.16.241.113
       50.16.241.114
       50.16.241.117
@@ -12,7 +14,8 @@ module Legitbot # :nodoc:
       52.5.190.19
       54.197.234.188
       54.208.100.253
-      23.21.227.69
+      54.208.102.37
+      107.21.1.8
     ]
   end
 

data/lib/legitbot/legitbot.rb

@@ -18,6 +18,7 @@ module Legitbot
   # otherwise.
   # :yields: a found bot
   #
+  # rubocop:disable Metrics/CyclomaticComplexity
   def self.bot(user_agent, ip)
     bots = @rules
            .select { |rule| rule[:fragments].any? { |f| user_agent.index f } }
@@ -32,6 +33,7 @@ module Legitbot
       selected
     end
   end
+  # rubocop:enable Metrics/CyclomaticComplexity
 
   def self.rule(clazz, fragments)
     @rules << { class: clazz, fragments: fragments }

data/lib/legitbot/oracle.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html
+  class Oracle < BotMatch
+    ip_ranges '148.64.56.0/24'
+  end
+
+  rule Legitbot::Oracle, %w[GrapeshotCrawler]
+end

data/lib/legitbot/twitter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started
+  # https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards
+  class Twitter < BotMatch
+    ip_ranges %w[
+      199.16.156.0/22
+      199.59.148.0/22
+    ]
+  end
+
+  rule Legitbot::Twitter, %w[Twitterbot]
+end

data/lib/legitbot/validators/ip_ranges.rb

@@ -59,22 +59,26 @@ module Legitbot
           partition_ips(@ip_ranges_loader.call)
         end
 
-        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
         def partition_ips(ips)
-          return [] if ips.empty?
+          return [] unless ips&.any?
 
           ips
             .map { |cidr| IPAddr.new(cidr) }
             .partition(&:ipv4?)
             .each_with_index
             .map do |list, index|
-              ranges = list.map(&:to_range).map do |r|
-                (r.begin.to_i..r.end.to_i)
-              end
-              [FAMILIES[index], IntervalTree::Tree.new(ranges)]
+              [FAMILIES[index], build_interval_tree(list)]
             end.to_h
         end
-        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+        private
+
+        def build_interval_tree(list)
+          ranges = list.map(&:to_range).map do |r|
+            (r.begin.to_i..r.end.to_i)
+          end
+          IntervalTree::Tree.new(ranges)
+        end
       end
     end
   end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '0.4.0'
+  VERSION = '1.0.1'
 end

data/lib/legitbot/yandex.rb

@@ -7,27 +7,40 @@ module Legitbot # :nodoc:
   end
 
   rule Legitbot::Yandex, %w[
-    YandexBot
     YandexAccessibilityBot
-    YandexMobileBot
-    YandexDirectDyn
-    YandexScreenshotBot
-    YandexImages
-    YandexVideo
-    YandexVideoParser
-    YandexMedia
+    YandexAdNet
     YandexBlogs
+    YandexBot/
+    YandexCalendar
+    YandexDirect/
+    YandexDirectDyn
     YandexFavicons
-    YandexWebmaster
-    YandexPagechecker
-    YandexImageResizer
     YaDirectFetcher
-    YandexCalendar
-    YandexSitelinks
+    YandexForDomain
+    YandexImages
+    YandexImageResizer
+    YandexMobileBot
+    YandexMarket
+    YandexMedia
     YandexMetrika
+    YandexMobileScreenShotBot
     YandexNews
-    YandexVertis
+    YandexOntoDB
+    YandexOntoDBAPI
+    YandexPagechecker
+    YandexPartner
+    YandexRCA
     YandexSearchShop
+    YandexSitelinks
+    YandexSpravBot
+    YandexTracker
+    YandexTurbo
+    YandexVertis
     YandexVerticals
+    YandexVideo
+    YandexVideoParser
+    YandexWebmaster
+    YandexScreenshotBot
+    YandexMedianaBot
   ]
 end

data/test/apple_test.rb

@@ -16,7 +16,7 @@ class AppleTest < Minitest::Test
     assert match.fake?, msg: "#{ip} is a fake Applebot IP"
   end
 
-  # rubocop:disable Metrics/LineLength
+  # rubocop:disable Layout/LineLength
   def test_user_agent
     bot = Legitbot.bot(
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)',
@@ -25,5 +25,5 @@ class AppleTest < Minitest::Test
     assert_equal :apple, bot.detected_as
     assert bot.valid?, msg: 'A valid Applebot User-agent and IP'
   end
-  # rubocop:enable Metrics/LineLength
+  # rubocop:enable Layout/LineLength
 end

data/test/facebook_test.rb

@@ -5,14 +5,14 @@ require 'legitbot'
 
 module Legitbot
   class Facebook
-    # rubocop:disable Metrics/LineLength
+    # rubocop:disable Layout/LineLength
     def self.whois
       {
         ipv4: ['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24'],
         ipv6: []
       }
     end
-    # rubocop:enable Metrics/LineLength
+    # rubocop:enable Layout/LineLength
   end
 end
 
@@ -33,7 +33,7 @@ class FacebookTest < Minitest::Test
     assert match.fake?, msg: "#{ip} is a fake Facebook IP"
   end
 
-  # rubocop:disable Metrics/LineLength, Metrics/MethodLength
+  # rubocop:disable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength
   def test_user_agent
     Legitbot.bot(
       'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
@@ -55,9 +55,9 @@ class FacebookTest < Minitest::Test
       'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0',
       '92.243.181.7'
     ) do |bot|
-      assert_equal :facebook, bot.detected_as
+      assert %i[facebook twitter].include?(bot.detected_as)
       assert bot.fake?, msg: 'fake Facebook'
     end
   end
-  # rubocop:enable Metrics/LineLength, Metrics/MethodLength
+  # rubocop:enable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength
 end

data/test/legitbot/validators/ip_ranges_test.rb

@@ -46,6 +46,11 @@ module Legitbot
       end
     end
 
+    class NilRanges
+      include IpRanges
+      ip_ranges { nil }
+    end
+
     class IpRangesTest < Minitest::Test
       def test_partition_method
         empty = NoRanges.partition_ips([])
@@ -108,6 +113,10 @@ module Legitbot
         assert_equal 2, LoadRanges.counter
       end
       # rubocop:enable Metrics/AbcSize
+
+      def test_nil_ranges
+        assert NilRanges.valid_ip?('127.0.0.1')
+      end
     end
   end
 end

data/test/oracle_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class OracleTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Oracle.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Oracle IP"
+  end
+
+  def test_valid_ip
+    ip = '148.64.56.64'
+    match = Legitbot::Oracle.new ip
+    assert match.valid?, msg: "#{ip} is a valid Oracle IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Oracle detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Oracle'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)',
+      '148.64.56.64'
+    )
+    assert bot, msg: 'Oracle detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Oracle'
+  end
+end

data/test/twitter_test.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'minitest/autorun'
+require 'legitbot'
+
+class TwitterTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Twitter.new ip
+    assert !match.valid?, msg: "#{ip} is not a real Twitter IP"
+  end
+
+  def test_valid_ip
+    ip = '199.16.156.125'
+    match = Legitbot::Twitter.new ip
+    assert match.valid?, msg: "#{ip} is a valid Twitter IP"
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Twitterbot/1.0',
+      '149.210.164.47'
+    )
+    assert bot, msg: 'Twitter detected from User-Agent'
+    assert !bot.valid?, msg: 'Not a valid Twitter'
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Twitterbot/1.0',
+      '199.16.156.125'
+    )
+    assert bot, msg: 'Twitter detected from User-Agent'
+    assert bot.valid?, msg: 'Valid Twitter'
+  end
+end

metadata

@@ -1,47 +1,62 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Alexander Azarov
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-18 00:00:00.000000000 Z
+date: 2020-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: irrc
+  name: augmented_interval_tree
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.1
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
-  name: augmented_interval_tree
+  name: irrc
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: bump
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
@@ -49,61 +64,83 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.0
+        version: 5.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.0
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.74.0
+        version: 12.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.74.0
+        version: 12.3.0
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.74'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.0
+        version: 0.74.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.74'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.1.0
-description: A library to make sure a Web request has been made by a real search engine,
-  not a malicious agent
+        version: 0.74.0
+description: Does Web request come from a real search engine or from an impersonating
+  agent?
 email: self@alaz.me
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/build.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
+- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -120,7 +157,9 @@ files:
 - lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
 - lib/legitbot/legitbot.rb
+- lib/legitbot/oracle.rb
 - lib/legitbot/pinterest.rb
+- lib/legitbot/twitter.rb
 - lib/legitbot/validators/domains.rb
 - lib/legitbot/validators/ip_ranges.rb
 - lib/legitbot/version.rb
@@ -134,12 +173,14 @@ files:
 - test/legitbot/validators/domains_test.rb
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb
+- test/oracle_test.rb
 - test/pinterest_test.rb
+- test/twitter_test.rb
 homepage: https://github.com/alaz/legitbot
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--charset=UTF-8"
 require_paths:
@@ -148,16 +189,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: 'Validate requests from Web crawlers: impersonating or not?'
 test_files:
@@ -168,6 +208,8 @@ test_files:
 - test/ahrefs_test.rb
 - test/apple_test.rb
 - test/apple_as_google_test.rb
+- test/oracle_test.rb
 - test/google_test.rb
 - test/botmatch_test.rb
 - test/facebook_test.rb
+- test/twitter_test.rb

data/.travis.yml

@@ -1,12 +0,0 @@
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.5
-  - 2.3
-  - jruby-head
-jdk:
-  - openjdk8
-before_install:
-  - gem update --system
-  - gem install bundler