legitbot 1.7.3 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eb23292d664c393d5b39a9d7a69b0469dbf326f1c5ef2cda2236cd16baf6eda
-  data.tar.gz: fefe1fd716e81ff8e0f38950be0790006cc84bb9b9c4b1a53997331010efc861
+  metadata.gz: 76098bb34095ff5b37ed3732b6f00baa6e8491b813f61faf0d717c0c35018885
+  data.tar.gz: 5ed3f6c8d09d019685e9a5ff33844de03b3bbf31b1a5156fc4e88264dbcc5d08
 SHA512:
-  metadata.gz: 7d156a7574284f8b401f6bcf78bb3f005f6f8c4ad11e67c9f30b515ed08d7855481052716cc2dd60f1eb33f19702b5b6bf8a5bd42b56c7907bc24d791cc4fa63
-  data.tar.gz: b33800ac92eaa874a4b74372281b3d512f34b6ab18e7743ef9857237508188ab92fdca1b554a0307397cf63d898f69c23f02462adbe8d52e0a58011cbda9ec4f
+  metadata.gz: 4dcd231f388e8134347db6c22286dfbf3e9155f5b79714c70f83fef8a1d2c2d3b990ea0b2d4f7beab4a59bfe0f1a326558d02b393340b628c55aebfd10cfdbb7
+  data.tar.gz: b66d66abe8eeace6c74bdfacab6cbe3dafa18c9c7c2325df313bc8a6842e81e510040dd1f7c1f0ff32c0692f75e976ff79089ac31a34125391d76d3bd2c2d8fc

data/Gemfile

@@ -2,3 +2,16 @@
 
 source 'https://rubygems.org'
 gemspec
+
+group :development do
+  gem 'bump'
+  gem 'dns_mock'
+  gem 'jsonpath'
+  gem 'minitest'
+  gem 'minitest-hooks'
+  gem 'nokogiri'
+  gem 'rake'
+  gem 'rubocop'
+  gem 'rubocop-minitest'
+  gem 'simplecov-cobertura'
+end

data/README.md

@@ -11,8 +11,8 @@ Suppose you have a Web request and you would like to check it is not diguised:
 bot = Legitbot.bot(userAgent, ip)
 ```
 
-`bot` will be `nil` if no bot signature was found in the `User-Agent`. Otherwise,
-it will be an object with methods
+`bot` will be `nil` if no bot signature was found in the `User-Agent`.
+Otherwise, it will be an object with methods
 
 ```ruby
 bot.detected_as # => :google
@@ -29,9 +29,9 @@ Rack::Attack.blocklist("fake Googlebot") do |req|
 end
 ```
 
-Or if you do not like all those ghoulish crawlers stealing your
-content, evaluating it and getting ready to invade your site with spammers,
-then block them all:
+Or if you do not like all those ghoulish crawlers stealing your content,
+evaluating it and getting ready to invade your site with spammers, then block
+them all:
 
 ```ruby
 Rack::Attack.blocklist 'fake search engines' do |request|
@@ -43,26 +43,31 @@ end
 
 [Semantic versioning](https://semver.org/) with the following clarifications:
 
-* MINOR version is incremented when support for new bots is added.
-* PATCH version is incremented when validation logic for a bot changes (IP list updated, for example).
+- MINOR version is incremented when support for new bots is added.
+- PATCH version is incremented when validation logic for a bot changes (IP list
+  updated, for example).
 
 ## Supported
 
-* [Ahrefs](https://ahrefs.com/robot)
-* [Alexa](https://support.alexa.com/hc/en-us/articles/360046707834-What-are-the-IP-addresses-for-Alexa-s-Certify-and-Site-Audit-crawlers-)
-* [Amazon AdBot](https://adbot.amazon.com/index.html)
-* [Applebot](https://support.apple.com/en-us/HT204683)
-* [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
-* [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
-* [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
-* [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
-* [Google crawlers](https://support.google.com/webmasters/answer/1061943)
-* [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
-* [Petal search engine](http://aspiegel.com/petalbot)
-* [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
-* [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started), the list of IPs is in the [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
-* [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
-* [You.com](https://about.you.com/youbot/)
+- [Ahrefs](https://ahrefs.com/robot)
+- [Alexa](https://support.alexa.com/hc/en-us/articles/360046707834-What-are-the-IP-addresses-for-Alexa-s-Certify-and-Site-Audit-crawlers-)
+- [Amazon AdBot](https://adbot.amazon.com/index.html)
+- [Applebot](https://support.apple.com/en-us/HT204683)
+- [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
+- [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
+- [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
+- [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
+- [Google crawlers](https://support.google.com/webmasters/answer/1061943)
+- [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)
+- [OpenAI GPTBot](https://platform.openai.com/docs/gptbot)
+- [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
+- [Petal search engine](http://aspiegel.com/petalbot)
+- [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
+- [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started),
+  the list of IPs is in the
+  [Troubleshooting page](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/troubleshooting-cards)
+- [Yandex robots](https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.xml)
+- [You.com](https://about.you.com/youbot/)
 
 ## License
 
@@ -70,16 +75,18 @@ Apache 2.0
 
 ## Other projects
 
-* Play Framework variant in Scala: [play-legitbot](https://github.com/osinka/play-legitbot)
-* Article [When (Fake) Googlebots Attack Your Rails App](http://jessewolgamott.com/blog/2015/11/17/when-fake-googlebots-attack-your-rails-app/)
-* [Voight-Kampff](https://github.com/biola/Voight-Kampff) is a Ruby gem that
+- Play Framework variant in Scala:
+  [play-legitbot](https://github.com/osinka/play-legitbot)
+- Article
+  [When (Fake) Googlebots Attack Your Rails App](http://jessewolgamott.com/blog/2015/11/17/when-fake-googlebots-attack-your-rails-app/)
+- [Voight-Kampff](https://github.com/biola/Voight-Kampff) is a Ruby gem that
   detects bots by `User-Agent`
-* [crawler_detect](https://github.com/loadkpi/crawler_detect) is a Ruby gem and Rack
-  middleware to detect crawlers by few different request headers, including `User-Agent`
-* Project Honeypot's
-  [http:BL](https://www.projecthoneypot.org/httpbl_api.php) can not only
-  classify IP as a search engine, but also label them as suspicious and
-  reports the number of days since the last activity. My implementation of
+- [crawler_detect](https://github.com/loadkpi/crawler_detect) is a Ruby gem and
+  Rack middleware to detect crawlers by few different request headers, including
+  `User-Agent`
+- Project Honeypot's [http:BL](https://www.projecthoneypot.org/httpbl_api.php)
+  can not only classify IP as a search engine, but also label them as suspicious
+  and reports the number of days since the last activity. My implementation of
   the protocol in Scala is [here](https://github.com/osinka/httpbl).
-* [CIDRAM](https://github.com/CIDRAM/CIDRAM) is a PHP routing manager with built-in support
-  to validate bots.
+- [CIDRAM](https://github.com/CIDRAM/CIDRAM) is a PHP routing manager with
+  built-in support to validate bots.

data/legitbot.gemspec

@@ -22,15 +22,6 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.6.0'
   spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
   spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
-  spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
-  spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
-  spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
-  spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
-  spec.add_development_dependency 'nokogiri', '~> 1.14', '>= 1.14.3'
-  spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.50.0', '>= 1.50.0'
-  spec.add_development_dependency 'rubocop-minitest', '~> 0.31.0', '>= 0.31.0'
-  spec.add_development_dependency 'simplecov-cobertura', '~> 2.0', '>= 2.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot/duckduckgo.rb

@@ -3,7 +3,7 @@
 module Legitbot # :nodoc:
   # https://duckduckgo.com/duckduckbot
   class DuckDuckGo < BotMatch
-    # @fetch:url https://help.duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
+    # @fetch:url https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/
     # @fetch:selector section.main article.content ul > li
     ip_ranges %w[
       20.185.79.15

data/lib/legitbot/gptbot.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://platform.openai.com/docs/gptbot
+  class GPTBot < BotMatch
+    # @fetch:url https://openai.com/gptbot-ranges.txt
+    ip_ranges %w[
+      20.15.240.64/28
+      20.15.240.80/28
+      20.15.240.96/28
+      20.15.240.176/28
+      20.15.241.0/28
+      20.15.242.128/28
+      20.15.242.144/28
+      20.15.242.192/28
+      40.83.2.64/28
+    ]
+  end
+
+  rule Legitbot::GPTBot, %w[GPTBot]
+end

data/lib/legitbot/ias.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/
+  class Ias < BotMatch
+    # @fetch:url https://integralads.com/policy-docs/iasbot.json
+    # @fetch:jsonpath $.publicIPs[*].ipv4
+    ip_ranges %w[
+      3.217.168.199
+      3.226.51.67
+      18.214.43.70
+    ]
+  end
+
+  rule Legitbot::Ias, %w[ias_crawler ias_wombles]
+end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.7.3'
+  VERSION = '1.9.0'
 end

data/lib/legitbot.rb

@@ -12,6 +12,8 @@ require_relative 'legitbot/bing'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
+require_relative 'legitbot/gptbot'
+require_relative 'legitbot/ias'
 require_relative 'legitbot/oracle'
 require_relative 'legitbot/petalbot'
 require_relative 'legitbot/pinterest'

data/lib/rubocop/cop/custom/ip_ranges.rb

@@ -3,6 +3,7 @@
 require 'ipaddr'
 require 'net/http'
 require 'nokogiri'
+require 'jsonpath'
 require 'rubocop'
 require 'uri'
 
@@ -24,8 +25,9 @@ module RuboCop
             params = fetch_params(node)
             return unless mandatory_params?(params)
 
-            existing_ips = read_node_ips value
-            new_ips = fetch_ips(**params)
+            existing_ips = normalise_list(read_node_ips(value))
+            new_ips = normalise_list(fetch_ips(**params))
+            return unless new_ips
             return if existing_ips == new_ips
 
             register_offense(value, new_ips, **params)
@@ -34,16 +36,46 @@ module RuboCop
 
         private
 
-        def fetch_ips(url:, selector:)
+        def fetch_ips(url:, selector: nil, jsonpath: nil)
+          body = get_url url
+          return unless body
+          return parse_html(body, selector) if selector
+          return parse_json(body, jsonpath) if jsonpath
+
+          parse_text(body)
+        end
+
+        def get_url(url)
           response = Net::HTTP.get_response URI(url)
+          unless response.is_a?(Net::HTTPOK)
+            add_global_offense "Could not fetch IPs from #{url} , HTTP status code #{response.code}"
+            return
+          end
+
           response.value
+          response.body
+        end
 
-          document = Nokogiri::HTML response.body
-          document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
+        def parse_html(body, selector)
+          document = Nokogiri::HTML body
+          document.css(selector).map(&:content)
+        end
+
+        def parse_json(body, jsonpath)
+          document = JSON.parse body
+          JsonPath.new(jsonpath).on(document)
+        end
+
+        def parse_text(body)
+          body.lines.map(&:chomp)
         end
 
         def read_node_ips(value)
-          value.child_nodes.map(&:value).sort_by(&IPAddr.method(:new))
+          value.child_nodes.map(&:value)
+        end
+
+        def normalise_list(ips)
+          ips.sort_by(&IPAddr.method(:new))
         end
 
         def register_offense(node, new_ips, **params)
@@ -54,7 +86,7 @@ module RuboCop
         end
 
         def mandatory_params?(params)
-          params.include?(:url) && params.include?(:selector)
+          params.include?(:url)
         end
 
         def fetch_params(node)

data/test/ias_test.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class IasTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Ias.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '18.214.43.70'
+    match = Legitbot::Ias.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)',
+      '18.214.43.72'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)',
+      '18.214.43.70'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.7.3
+  version: 1.9.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-06 00:00:00.000000000 Z
+date: 2023-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -50,186 +50,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.1
-- !ruby/object:Gem::Dependency
-  name: bump
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-- !ruby/object:Gem::Dependency
-  name: dns_mock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.1.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.1.0
-- !ruby/object:Gem::Dependency
-  name: minitest-hooks
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.14'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.14.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.14'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.14.3
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 13.0.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 13.0.0
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-- !ruby/object:Gem::Dependency
-  name: rubocop-minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-- !ruby/object:Gem::Dependency
-  name: simplecov-cobertura
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.0'
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -261,6 +81,8 @@ files:
 - lib/legitbot/duckduckgo.rb
 - lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
+- lib/legitbot/gptbot.rb
+- lib/legitbot/ias.rb
 - lib/legitbot/legitbot.rb
 - lib/legitbot/oracle.rb
 - lib/legitbot/petalbot.rb
@@ -284,6 +106,7 @@ files:
 - test/botmatch_test.rb
 - test/facebook_test.rb
 - test/google_test.rb
+- test/ias_test.rb
 - test/legitbot/validators/domains_test.rb
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb