legitbot 1.7.3 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eb23292d664c393d5b39a9d7a69b0469dbf326f1c5ef2cda2236cd16baf6eda
-  data.tar.gz: fefe1fd716e81ff8e0f38950be0790006cc84bb9b9c4b1a53997331010efc861
+  metadata.gz: a7cde94cf9e8a396867e4e97a490c1b4da0b300652e619ac200477a5e3aed1d5
+  data.tar.gz: 1d505df51aa086231f85080fcf3a60291a660cfce3ce79bdeab0591ba9d27578
 SHA512:
-  metadata.gz: 7d156a7574284f8b401f6bcf78bb3f005f6f8c4ad11e67c9f30b515ed08d7855481052716cc2dd60f1eb33f19702b5b6bf8a5bd42b56c7907bc24d791cc4fa63
-  data.tar.gz: b33800ac92eaa874a4b74372281b3d512f34b6ab18e7743ef9857237508188ab92fdca1b554a0307397cf63d898f69c23f02462adbe8d52e0a58011cbda9ec4f
+  metadata.gz: 41f811fd8c20c9a442218e36a8e54e1e72731b8443c6141205b5f3b7accbdbaeb85491134d2ca3f4cb817d01dc774348c52dae705a7655d98345f975782b5c4a
+  data.tar.gz: 77912f09be50c5d868099a6ecc2402b582d17039a82b9a57ee790724bf3b99123d7b5db071928f944c814afcc226818530b953c62f31d6f4a5d1df6bfdcefc54

data/Gemfile

@@ -2,3 +2,16 @@
 
 source 'https://rubygems.org'
 gemspec
+
+group :development do
+  gem 'bump'
+  gem 'dns_mock'
+  gem 'jsonpath'
+  gem 'minitest'
+  gem 'minitest-hooks'
+  gem 'nokogiri'
+  gem 'rake'
+  gem 'rubocop'
+  gem 'rubocop-minitest'
+  gem 'simplecov-cobertura'
+end

data/README.md

@@ -57,6 +57,7 @@ end
 * [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
 * [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
 * [Google crawlers](https://support.google.com/webmasters/answer/1061943)
+* [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)
 * [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
 * [Petal search engine](http://aspiegel.com/petalbot)
 * [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)

data/legitbot.gemspec

@@ -22,15 +22,6 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.6.0'
   spec.add_dependency 'fast_interval_tree', '~> 0.2', '>= 0.2.2'
   spec.add_dependency 'irrc', '~> 0.2', '>= 0.2.1'
-  spec.add_development_dependency 'bump', '~> 0.8', '>= 0.8.0'
-  spec.add_development_dependency 'dns_mock', '~> 1.5.0', '>= 1.5.0'
-  spec.add_development_dependency 'minitest', '~> 5.1', '>= 5.1.0'
-  spec.add_development_dependency 'minitest-hooks', '~> 1.5', '>= 1.5.0'
-  spec.add_development_dependency 'nokogiri', '~> 1.14', '>= 1.14.3'
-  spec.add_development_dependency 'rake', '~> 13.0', '>= 13.0.0'
-  spec.add_development_dependency 'rubocop', '~> 1.50.0', '>= 1.50.0'
-  spec.add_development_dependency 'rubocop-minitest', '~> 0.31.0', '>= 0.31.0'
-  spec.add_development_dependency 'simplecov-cobertura', '~> 2.0', '>= 2.0'
 
   spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
   spec.rdoc_options = ['--charset=UTF-8']

data/lib/legitbot/ias.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/
+  class Ias < BotMatch
+    # @fetch:url https://integralads.com/policy-docs/iasbot.json
+    # @fetch:jsonpath $.publicIPs[*].ipv4
+    ip_ranges %w[
+      3.217.168.199
+      3.226.51.67
+      18.214.43.70
+    ]
+  end
+
+  rule Legitbot::Ias, %w[ias_crawler ias_wombles]
+end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.7.3'
+  VERSION = '1.8.0'
 end

data/lib/legitbot.rb

@@ -12,6 +12,7 @@ require_relative 'legitbot/bing'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
+require_relative 'legitbot/ias'
 require_relative 'legitbot/oracle'
 require_relative 'legitbot/petalbot'
 require_relative 'legitbot/pinterest'

data/lib/rubocop/cop/custom/ip_ranges.rb

@@ -3,6 +3,7 @@
 require 'ipaddr'
 require 'net/http'
 require 'nokogiri'
+require 'jsonpath'
 require 'rubocop'
 require 'uri'
 
@@ -34,12 +35,17 @@ module RuboCop
 
         private
 
-        def fetch_ips(url:, selector:)
+        def fetch_ips(url:, selector: nil, jsonpath: nil)
           response = Net::HTTP.get_response URI(url)
           response.value
 
-          document = Nokogiri::HTML response.body
-          document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
+          if selector
+            document = Nokogiri::HTML response.body
+            document.css(selector).map(&:content).sort_by(&IPAddr.method(:new))
+          else
+            document = JSON.parse response.body
+            JsonPath.new(jsonpath).on(document).sort_by(&IPAddr.method(:new))
+          end
         end
 
         def read_node_ips(value)
@@ -54,7 +60,7 @@ module RuboCop
         end
 
         def mandatory_params?(params)
-          params.include?(:url) && params.include?(:selector)
+          params.include?(:url) && (params.include?(:selector) || params.include?(:jsonpath))
         end
 
         def fetch_params(node)

data/test/ias_test.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class IasTest < Minitest::Test
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::Ias.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '18.214.43.70'
+    match = Legitbot::Ias.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)',
+      '18.214.43.72'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'IAS Crawler (ias_crawler; http://integralads.com/site-indexing-policy/)',
+      '18.214.43.70'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.7.3
+  version: 1.8.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-06 00:00:00.000000000 Z
+date: 2023-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -50,186 +50,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.1
-- !ruby/object:Gem::Dependency
-  name: bump
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.8.0
-- !ruby/object:Gem::Dependency
-  name: dns_mock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.1.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.1.0
-- !ruby/object:Gem::Dependency
-  name: minitest-hooks
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.5.0
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.14'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.14.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.14'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.14.3
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 13.0.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 13.0.0
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.50.0
-- !ruby/object:Gem::Dependency
-  name: rubocop-minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.31.0
-- !ruby/object:Gem::Dependency
-  name: simplecov-cobertura
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.0'
 description: Does Web request come from a real search engine or from an impersonating
   agent?
 email: self@alaz.me
@@ -261,6 +81,7 @@ files:
 - lib/legitbot/duckduckgo.rb
 - lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
+- lib/legitbot/ias.rb
 - lib/legitbot/legitbot.rb
 - lib/legitbot/oracle.rb
 - lib/legitbot/petalbot.rb
@@ -284,6 +105,7 @@ files:
 - test/botmatch_test.rb
 - test/facebook_test.rb
 - test/google_test.rb
+- test/ias_test.rb
 - test/legitbot/validators/domains_test.rb
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb