legitbot 1.10.5 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ac43bac7745d6e0ef906342c82102e87c8644a6cd8a8af3077831f13ebbd79e
-  data.tar.gz: 0eeed734eadee6c380d41a604fa21e83aba8d8431cb8444b27fa2b0ec0cabe2a
+  metadata.gz: ccf22fe1fab3a7cab2955709eb3b0ada75a66305b255a36ec795eb092d9741c8
+  data.tar.gz: 6b03643eb517f59626c7a1e59b04f6808ce067b87597bf5f7e8486f4c1eb309c
 SHA512:
-  metadata.gz: 36b25cf91d14fd4211e31c9dd92f8ec7439155a2e7862329d848615ceaff52f5fd1800b56fb25f0956b01572ed019f4689a81de9ba1b828c696710f451517344
-  data.tar.gz: 49fbc215737f0d863c1a5ffdf28f5a41f438d31923987cfdbe58a60dd6056f20bc37985ebf1dca2e218dc30ab0a216ba005a670e2027f780280899e689ea7c0c
+  metadata.gz: 44f09102368337c185aa95c32a76dd551a01bf9fda2e098757383101bb3c57cced58f85fcad00f7c86dda1550df142b0a9549ea2d7c9c54c41cb2fffe7bfbabe
+  data.tar.gz: 6149c2e4eca68be2224ee529184a4df492fd224602aed1c08aa0a80af5b77ac902e23258331099c769ea4cb80f07c4b99df00526626d328fa13852afedb57eb0

data/README.md

@@ -50,17 +50,20 @@ end
 ## Supported
 
 - [Ahrefs](https://ahrefs.com/robot)
+- [Amazonbot](https://developer.amazon.com/amazonbot)
 - [Amazon AdBot](https://adbot.amazon.com/index.html)
-- [Applebot](https://support.apple.com/en-us/HT204683)
+- [Applebot](https://support.apple.com/en-us/119829)
 - [Baidu spider](http://help.baidu.com/question?prod_en=master&class=498&id=1000973)
 - [Bingbot](https://blogs.bing.com/webmaster/2012/08/31/how-to-verify-that-bingbot-is-bingbot/)
+- [BLEXBot (WebMeUp)](http://webmeup-crawler.com/)
+- [DataForSEO](https://dataforseo.com/dataforseo-bot)
 - [DuckDuckGo bot](https://duckduckgo.com/duckduckbot)
-- [Facebook crawler](https://developers.facebook.com/docs/sharing/webmasters/crawler)
 - [Google crawlers](https://support.google.com/webmasters/answer/1061943)
 - [IAS](https://integralads.com/ias-privacy-data-management/policies/site-indexing-policy/)
 - [OpenAI GPTBot](https://platform.openai.com/docs/gptbot)
 - [Oracle Data Cloud Crawler](https://www.oracle.com/corporate/acquisitions/grapeshot/crawler.html)
 - [Marginalia](https://www.marginalia.nu/marginalia-search/for-webmasters/)
+- [Meta / Facebook Web crawlers](https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/)
 - [Petal search engine](http://aspiegel.com/petalbot)
 - [Pinterest](https://help.pinterest.com/en/articles/about-pinterest-crawler-0)
 - [Twitterbot](https://developer.twitter.com/en/docs/tweets/optimize-with-cards/guides/getting-started),

data/lib/legitbot/amazon.rb

@@ -2,9 +2,10 @@
 
 module Legitbot # :nodoc:
   # https://adbot.amazon.com/index.html
+  # https://developer.amazon.com/amazonbot
   class Amazon < BotMatch
-    domains 'amazonadbot.com.'
+    domains 'amazon.', 'amazonadbot.com.'
   end
 
-  rule Legitbot::Amazon, %w[AmazonAdBot]
+  rule Legitbot::Amazon, %w[Amazonbot AmazonAdBot]
 end

data/lib/legitbot/apple.rb

@@ -3,10 +3,13 @@
 require 'ipaddr'
 
 module Legitbot # :nodoc:
-  # https://support.apple.com/en-us/HT204683
+  # https://support.apple.com/en-us/119829
   class Apple < BotMatch
-    ip_ranges '17.0.0.0/8'
+    domains 'applebot.apple.com.'
   end
 
-  rule Legitbot::Apple, %w[Applebot]
+  rule Legitbot::Apple, %w[
+    Applebot
+    iTMS
+  ]
 end

data/lib/legitbot/blexbot.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # http://webmeup-crawler.com/
+  class BLEXBot < BotMatch
+    domains 'webmeup.com.'
+  end
+
+  rule Legitbot::BLEXBot, %w[BLEXBot]
+end

data/lib/legitbot/dataforseo.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://dataforseo.com/dataforseo-bot
+  class DataForSEO < BotMatch
+    domains 'dataforseo.com.'
+  end
+
+  rule Legitbot::DataForSEO, %w[DataForSeoBot]
+end

data/lib/legitbot/facebook.rb

@@ -1,20 +1,14 @@
 # frozen_string_literal: true
 
-require 'irrc'
+require_relative 'meta'
 
 module Legitbot # :nodoc:
   # https://developers.facebook.com/docs/sharing/webmasters/crawler
   class Facebook < BotMatch
-    AS = 'AS32934'
+    extend MetaIpRanges
 
     ip_ranges do
-      client = Irrc::Client.new
-      client.query :radb, AS, source: :radb
-      results = client.perform
-
-      %i[ipv4 ipv6].map do |family|
-        results[AS][family][AS]
-      end.flatten
+      fetch_ip_ranges
     end
   end
 

data/lib/legitbot/meta.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'irrc'
+
+module Legitbot # :nodoc:
+  module MetaIpRanges # :nodoc:
+    AS = 'AS32934'
+
+    def fetch_ip_ranges
+      client = Irrc::Client.new
+      client.query :radb, AS, source: :radb
+      results = client.perform
+
+      %i[ipv4 ipv6].map do |family|
+        results[AS][family][AS]
+      end.flatten
+    end
+  end
+
+  # https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/
+  class Meta < BotMatch
+    extend MetaIpRanges
+
+    ip_ranges do
+      fetch_ip_ranges
+    end
+  end
+
+  rule Legitbot::Meta, %w[
+    meta-externalagent
+    meta-externalfetcher
+  ]
+end

data/lib/legitbot/openai.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Legitbot # :nodoc:
+  # https://platform.openai.com/docs/gptbot
+  class GPTBot < BotMatch
+    # NOTE: fetching is disabled, see #131
+    # @ fetch:url https://openai.com/gptbot.json
+    ip_ranges %w[
+      20.171.206.0/24
+      52.230.152.0/24
+      52.233.106.0/24
+    ]
+  end
+
+  # https://platform.openai.com/docs/bots
+  class OpenAIChat < BotMatch
+    # NOTE: fetching is disabled, see #131
+    # @ fetch:url https://openai.com/chatgpt-user.json
+    ip_ranges %w[
+      23.98.142.176/28
+      40.84.180.224/28
+      13.65.240.240/28
+      20.97.189.96/28
+      20.161.75.208/28
+      52.225.75.208/28
+      52.156.77.144/28
+      40.84.221.208/28
+      40.84.221.224/28
+      40.84.180.64/28
+    ]
+  end
+
+  # https://platform.openai.com/docs/bots
+  class OpenAISearch < BotMatch
+    # NOTE: fetching is disabled, see #131
+    # @ fetch:url https://openai.com/searchbot.json
+    ip_ranges %w[
+      20.42.10.176/28
+      172.203.190.128/28
+    ]
+  end
+
+  rule Legitbot::GPTBot, %w[GPTBot]
+  rule Legitbot::OpenAIChat, %w[ChatGPT-User]
+  rule Legitbot::OpenAISearch, %w[OAI-SearchBot]
+end

data/lib/legitbot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Legitbot
-  VERSION = '1.10.5'
+  VERSION = '1.11.0'
 end

data/lib/legitbot.rb

@@ -8,13 +8,16 @@ require_relative 'legitbot/amazon'
 require_relative 'legitbot/apple'
 require_relative 'legitbot/baidu'
 require_relative 'legitbot/bing'
+require_relative 'legitbot/blexbot'
+require_relative 'legitbot/dataforseo'
 require_relative 'legitbot/duckduckgo'
 require_relative 'legitbot/facebook'
 require_relative 'legitbot/google'
-require_relative 'legitbot/gptbot'
 require_relative 'legitbot/ias'
+require_relative 'legitbot/openai'
 require_relative 'legitbot/oracle'
 require_relative 'legitbot/marginalia'
+require_relative 'legitbot/meta'
 require_relative 'legitbot/petalbot'
 require_relative 'legitbot/pinterest'
 require_relative 'legitbot/twitter'

data/test/amazon_test.rb

@@ -30,7 +30,7 @@ class AmazonTest < Minitest::Test
     refute_predicate bot, :valid?
   end
 
-  def test_valid_ua
+  def test_user_agent1
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
       '54.166.7.90'
@@ -40,7 +40,19 @@ class AmazonTest < Minitest::Test
     assert_predicate bot, :valid?
   end
 
-  def test_valid_name
+  # rubocop:disable Layout/LineLength
+  def test_user_agent2
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
+      '52.70.240.171'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+  # rubocop:enable Layout/LineLength
+
+  def test_valid_name1
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',
       '54.166.7.90'
@@ -49,6 +61,17 @@ class AmazonTest < Minitest::Test
     assert_equal :amazon, bot.detected_as
   end
 
+  # rubocop:disable Layout/LineLength
+  def test_valid_name2
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
+      '52.70.240.171'
+    )
+
+    assert_equal :amazon, bot.detected_as
+  end
+  # rubocop:enable Layout/LineLength
+
   def test_fake_name
     bot = Legitbot.bot(
       'Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)',

data/test/blexbot_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class BLEXBot < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::BLEXBot.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '65.21.113.197'
+    match = Legitbot::BLEXBot.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '149.210.164.47'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '65.21.113.197'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '65.21.113.197'
+    )
+
+    assert_equal :blexbot, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)',
+      '81.1.172.108'
+    )
+
+    assert_equal :blexbot, bot.detected_as
+  end
+end

data/test/dataforseo_test.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class DataForSEOTest < Minitest::Test
+  include Minitest::Hooks
+  include DnsServerMock
+
+  def test_malicious_ip
+    ip = '149.210.164.47'
+    match = Legitbot::DataForSEO.new ip
+
+    refute_predicate match, :valid?
+  end
+
+  def test_valid_ip
+    ip = '136.243.228.176'
+    match = Legitbot::DataForSEO.new ip
+
+    assert_predicate match, :valid?
+  end
+
+  def test_malicious_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '149.210.164.47'
+    )
+
+    assert bot
+    refute_predicate bot, :valid?
+  end
+
+  def test_valid_ua
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '136.243.228.176'
+    )
+
+    assert bot
+    assert_predicate bot, :valid?
+  end
+
+  def test_valid_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '136.243.228.176'
+    )
+
+    assert_equal :dataforseo, bot.detected_as
+  end
+
+  def test_fake_name
+    bot = Legitbot.bot(
+      'Mozilla/5.0 (compatible; DataForSeoBot; +https://dataforseo.com/dataforseo-bot)',
+      '81.1.172.108'
+    )
+
+    assert_equal :dataforseo, bot.detected_as
+  end
+end

data/test/facebook_test.rb

@@ -2,19 +2,6 @@
 
 require_relative 'test_helper'
 
-module Legitbot
-  class Facebook
-    # rubocop:disable Layout/LineLength
-    def self.whois
-      {
-        ipv4: ['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24'],
-        ipv6: []
-      }
-    end
-    # rubocop:enable Layout/LineLength
-  end
-end
-
 class FacebookTest < Minitest::Test
   def test_valid_ip
     ip = '69.63.186.89'

data/test/lib/dns_server_mock.rb

@@ -29,6 +29,12 @@ TEST_DNS_RECORDS = {
   '54.166.7.90' => {
     ptr: %w[crawler-54-166-7-90.amazonadbot.com]
   },
+  '52-70-240-171.crawl.amazonbot.amazon' => {
+    a: %w[52.70.240.171]
+  },
+  '52.70.240.171' => {
+    ptr: %w[52-70-240-171.crawl.amazonbot.amazon]
+  },
 
   # Apple
   '17-58-98-60.applebot.apple.com' => {
@@ -38,6 +44,21 @@ TEST_DNS_RECORDS = {
     ptr: %w[17-58-98-60.applebot.apple.com]
   },
 
+  # BLEXBot (WebMeUp)
+  'pot22.webmeup.com' => {
+    a: %w[65.21.113.197]
+  },
+  '65.21.113.197' => {
+    ptr: %w[pot22.webmeup.com]
+  },
+  # DataForSEO
+  'crawling-gateway-136-243-228-176.dataforseo.com' => {
+    a: %w[136.243.228.176]
+  },
+  '136.243.228.176' => {
+    ptr: %w[crawling-gateway-136-243-228-176.dataforseo.com]
+  },
+
   # Google
   'crawl-66-249-64-141.googlebot.com' => {
     a: %w[66.249.64.141]

data/test/lib/meta_ip_ranges_mock.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Legitbot
+  module MetaIpRanges
+    alias fetch_ip_ranges_orig fetch_ip_ranges
+
+    # rubocop:disable Layout/LineLength
+    def fetch_ip_ranges
+      ['69.63.176.0/20', '66.220.144.0/20', '66.220.144.0/21', '69.63.184.0/21', '69.63.176.0/21', '74.119.76.0/22', '69.171.255.0/24', '173.252.64.0/18', '69.171.224.0/19', '69.171.224.0/20', '103.4.96.0/22', '69.63.176.0/24', '173.252.64.0/19', '173.252.70.0/24', '31.13.64.0/18', '31.13.24.0/21', '66.220.152.0/21', '66.220.159.0/24', '69.171.239.0/24', '69.171.240.0/20', '31.13.64.0/19', '31.13.64.0/24', '31.13.65.0/24', '31.13.67.0/24', '31.13.68.0/24', '31.13.69.0/24', '31.13.70.0/24', '31.13.71.0/24', '31.13.72.0/24', '31.13.73.0/24', '31.13.74.0/24', '31.13.75.0/24', '31.13.76.0/24', '31.13.77.0/24', '31.13.96.0/19', '31.13.66.0/24', '173.252.96.0/19', '69.63.178.0/24', '31.13.78.0/24', '31.13.79.0/24', '31.13.80.0/24', '31.13.82.0/24', '31.13.83.0/24', '31.13.84.0/24', '31.13.85.0/24', '31.13.86.0/24', '31.13.87.0/24', '31.13.88.0/24', '31.13.89.0/24', '31.13.90.0/24', '31.13.91.0/24', '31.13.92.0/24', '31.13.93.0/24', '31.13.94.0/24', '31.13.95.0/24', '69.171.253.0/24', '69.63.186.0/24', '31.13.81.0/24', '179.60.192.0/22', '179.60.192.0/24', '179.60.193.0/24', '179.60.194.0/24', '179.60.195.0/24', '185.60.216.0/22', '45.64.40.0/22', '185.60.216.0/24', '185.60.217.0/24', '185.60.218.0/24', '185.60.219.0/24', '129.134.0.0/16', '157.240.0.0/16', '157.240.8.0/24', '157.240.0.0/24', '157.240.1.0/24', '157.240.2.0/24', '157.240.3.0/24', '157.240.4.0/24', '157.240.5.0/24', '157.240.6.0/24', '157.240.7.0/24', '157.240.9.0/24', '157.240.10.0/24', '157.240.16.0/24', '157.240.19.0/24', '157.240.11.0/24', '157.240.12.0/24', '157.240.13.0/24', '157.240.14.0/24', '157.240.15.0/24', '157.240.17.0/24', '157.240.18.0/24', '157.240.20.0/24', '157.240.21.0/24', '157.240.22.0/24', '157.240.23.0/24', '157.240.0.0/17', '69.171.250.0/24', '157.240.24.0/24', '157.240.25.0/24', '199.201.64.0/24', '199.201.65.0/24', '199.201.64.0/22', '204.15.20.0/22', '157.240.192.0/24', '129.134.0.0/17', '157.240.198.0/24']
+    end
+    # rubocop:enable Layout/LineLength
+  end
+end

data/test/meta_test.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative 'test_helper'
+
+class MetaIpRanges
+  include Legitbot::MetaIpRanges
+end
+
+class MetaTest < Minitest::Test
+  def test_fetch_ips
+    # NOTE: network call
+    ip_ranges = MetaIpRanges.new.fetch_ip_ranges_orig
+
+    refute_nil ip_ranges
+    assert_kind_of Array, ip_ranges
+    refute_empty ip_ranges
+  end
+
+  def test_valid_ip
+    ip = '69.63.186.89'
+    match = Legitbot::Meta.new(ip)
+
+    assert_predicate match, :valid?
+
+    ip = '69.171.251.1'
+    match = Legitbot::Meta.new(ip)
+
+    assert_predicate match, :valid?
+  end
+
+  def test_invalid_ip
+    ip = '127.0.0.1'
+    match = Legitbot::Meta.new(ip)
+
+    assert_predicate match, :fake?
+  end
+
+  def test_user_agent1
+    Legitbot.bot(
+      'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)',
+      '31.13.76.56'
+    ) do |bot|
+      assert_equal :meta, bot.detected_as
+      assert_predicate bot, :valid?
+    end
+  end
+
+  def test_user_agent2
+    Legitbot.bot(
+      'meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)',
+      '173.252.87.8'
+    ) do |bot|
+      assert_equal :meta, bot.detected_as
+      assert_predicate bot, :valid?
+    end
+  end
+
+  def test_user_agent3
+    Legitbot.bot(
+      'meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)',
+      '173.252.87.8'
+    ) do |bot|
+      assert_equal :meta, bot.detected_as
+      assert_predicate bot, :valid?
+    end
+  end
+
+  # rubocop:disable Layout/LineLength
+  def test_user_agent4
+    Legitbot.bot(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 meta-externalagent/1.1 Twitterbot/1.0',
+      '92.243.181.7'
+    ) do |bot|
+      assert_includes %i[meta twitter], bot.detected_as
+      assert_predicate bot, :fake?
+    end
+  end
+  # rubocop:enable Layout/LineLength
+end

data/test/test_helper.rb

@@ -13,3 +13,4 @@ require 'legitbot'
 require 'minitest/autorun'
 require 'minitest/hooks/test'
 require 'lib/dns_server_mock'
+require 'lib/meta_ip_ranges_mock'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legitbot
 version: !ruby/object:Gem::Version
-  version: 1.10.5
+  version: 1.11.0
 platform: ruby
 authors:
 - Alexander Azarov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-17 00:00:00.000000000 Z
+date: 2024-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast_interval_tree
@@ -76,15 +76,18 @@ files:
 - lib/legitbot/apple.rb
 - lib/legitbot/baidu.rb
 - lib/legitbot/bing.rb
+- lib/legitbot/blexbot.rb
 - lib/legitbot/botmatch.rb
 - lib/legitbot/config/resolver.rb
+- lib/legitbot/dataforseo.rb
 - lib/legitbot/duckduckgo.rb
 - lib/legitbot/facebook.rb
 - lib/legitbot/google.rb
-- lib/legitbot/gptbot.rb
 - lib/legitbot/ias.rb
 - lib/legitbot/legitbot.rb
 - lib/legitbot/marginalia.rb
+- lib/legitbot/meta.rb
+- lib/legitbot/openai.rb
 - lib/legitbot/oracle.rb
 - lib/legitbot/petalbot.rb
 - lib/legitbot/pinterest.rb
@@ -103,7 +106,9 @@ files:
 - test/ahrefs_test.rb
 - test/amazon_test.rb
 - test/apple_test.rb
+- test/blexbot_test.rb
 - test/botmatch_test.rb
+- test/dataforseo_test.rb
 - test/facebook_test.rb
 - test/google_test.rb
 - test/ias_test.rb
@@ -111,6 +116,8 @@ files:
 - test/legitbot/validators/ip_ranges_test.rb
 - test/legitbot_test.rb
 - test/lib/dns_server_mock.rb
+- test/lib/meta_ip_ranges_mock.rb
+- test/meta_test.rb
 - test/oracle_test.rb
 - test/petalbot_test.rb
 - test/pinterest_test.rb

data/lib/legitbot/gptbot.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Legitbot # :nodoc:
-  # https://platform.openai.com/docs/gptbot
-  class GPTBot < BotMatch
-    # NOTE: fetching has been disabled, see #131
-    # @ fetch:url https://openai.com/gptbot-ranges.txt
-    ip_ranges %w[
-      52.230.152.0/24
-      52.233.106.0/24
-    ]
-  end
-
-  rule Legitbot::GPTBot, %w[GPTBot]
-end