drive 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf5ec0cbf839ae0e64fabcecc4f66f40a1e4654252053c57837c8895b290366f
-  data.tar.gz: e5819d0ad8774657ad71eb4695971399ac22215fd7c37d96cf7cc8db3c64f543
+  metadata.gz: 99a4adcee91a1d02137f10f2ffff028ee3bcabd96896d550c71e81c443ad47b8
+  data.tar.gz: 2a5e5b329808e3e315464d70e1e8fe1357a5736632d96dde4077f3f36a57518e
 SHA512:
-  metadata.gz: c2589ac5dc34686550476cf266d06ef8424c0e3d8307761cb4861d15bc3cd7b00884407ac273132f4efa70b4ec1cc671cec67b0bbec70e426aa53d65dbd3bfd3
-  data.tar.gz: e4610a657c780d8458a236bc7512b8dc984e75337e4a17624cff92cc53458799375761027fa5361432c924f0fdfaa088ac5dcde2eddf012039d746ba7e4945f9
+  metadata.gz: 57b24c0022b7476eb3db58ab1108ee86283f5397a49704c39626f751c8158baad8d35404ceebafedaf53bbefc912d5a99e65ddbe6cd6cfea546b8417b20320d8
+  data.tar.gz: eeaa76fd2453f73da3332ded46d662d4165b458ea9db7359582ec6c05b980db96446eaccf47ed880893d1f4298fa37876c9fb573a7b9d399953d43daa98a29a4

data/CHANGELOG.md

@@ -6,6 +6,15 @@ For more information about changelogs, check [Keep a Changelog](http://keepachan
 
 ## [Unreleased]
 
+## 0.5.0 - 2026-10-07
+
+* [FEATURE] `RateLimited` no longer limits Claude's, ChatGPT's or Perplexity's search crawler
+
+  `Crawler.verified?(request)` believes a user agent naming Claude-SearchBot, OAI-SearchBot or
+  PerplexityBot, as well as Google's, Bing's or Apple's crawler, only where its address is in the
+  ranges its search engine publishes as JSON, fetched once a day. None of the three new ones
+  resolves to a domain of its own, so no crawler is checked by DNS any more.
+
 ## 0.4.0 - 2026-10-06
 
 * [BREAKING] No more log of changes, and no feed

data/README.md

@@ -11,8 +11,8 @@ files, a monthly database — changing only the rows the source did, and countin
 - **`AccessFile`**: stages a table of a Microsoft Access file, read with `mdb-export`.
 - **`RateLimited`**: 20 requests a minute and 200 a day per address, for public pages. A search
   engine's own crawler has no limit.
-- **`Crawler`**: whether a request comes from Google's, Bing's or Apple's crawler: its address
-  resolves to their domain and back, checked once a day per address.
+- **`Crawler`**: whether a request comes from the crawler of Google, Bing, Apple, Claude, ChatGPT
+  or Perplexity: its address is in the ranges they publish, fetched once a day.
 
 It needs PostgreSQL.
 
@@ -24,10 +24,10 @@ gem install drive
 
 ```ruby
 # Gemfile
-gem 'drive', '~> 0.4.0'
+gem 'drive', '~> 0.5.0'
 ```
 
-`~> 0.4.0` takes every 0.4.x and stops short of 0.5, which may break what 0.4 does.
+`~> 0.5.0` takes every 0.5.x and stops short of 0.6, which may break what 0.5 does.
 
 Then copy its migrations, of runs, and run them:
 

data/app/models/crawler.rb

@@ -1,33 +1,34 @@
-require 'resolv'
+require 'ipaddr'
+require 'net/http'
 
 # A search engine's own crawler, told apart from a client that only names one in its user agent:
-# its address resolves to the search engine's domain, and that host back to the address, which is
-# how Google and Bing say to check theirs.
+# its address is in the ranges the search engine publishes for it.
 module Crawler
-  # The user agents of the crawlers checked.
-  AGENTS = /Googlebot|Google-InspectionTool|Storebot-Google|GoogleOther|bingbot|Applebot/i
-
-  # The domains their addresses resolve to.
-  DOMAINS = %w[.googlebot.com .google.com .search.msn.com .applebot.apple.com].freeze
+  # The ranges each search engine publishes, by the user agents of its crawlers.
+  RANGES = {
+    /Googlebot|Google-InspectionTool|Storebot-Google|GoogleOther/i =>
+      'https://developers.google.com/static/crawling/ipranges/common-crawlers.json',
+    /bingbot/i => 'https://www.bing.com/toolbox/bingbot.json',
+    /Applebot/i => 'https://search.developer.apple.com/applebot.json',
+    /Claude-SearchBot/i => 'https://claude.com/crawling/bots.json',
+    /OAI-SearchBot/i => 'https://openai.com/searchbot.json',
+    /PerplexityBot/i => 'https://www.perplexity.ai/perplexitybot.json',
+  }.freeze
 
   # @param request [ActionDispatch::Request] request a page was asked with.
-  # @return [Boolean] whether it came from a crawler, checked once a day for each address.
+  # @return [Boolean] whether it came from a crawler, its ranges fetched once a day.
   def self.verified?(request)
-    ip = request.remote_ip
-    request.user_agent.to_s.match?(AGENTS) &&
-      Rails.cache.fetch(['crawler', ip], expires_in: 1.day) { resolves? ip }
+    url = RANGES.find { |agent, _| request.user_agent.to_s.match? agent }&.last
+    url.present? && ranges(url).any? { it.include? request.remote_ip }
   end
 
-  # @param ip [String] address a request came from.
-  # @return [Boolean] whether it names a crawler's host, and that host names it back.
-  def self.resolves?(ip)
-    dns = Resolv::DNS.new.tap { it.timeouts = 2 }
-    host = dns.getname(ip).to_s
-    DOMAINS.any? { host.end_with? it } && dns.getaddresses(host).map(&:to_s).include?(ip)
-  rescue Resolv::ResolvError
-    false
-  ensure
-    dns&.close
+  private_class_method def self.ranges(url)
+    Rails.cache.fetch(['crawler', url], expires_in: 1.day) do
+      uri = URI url
+      body = Net::HTTP.start(uri.host, use_ssl: true, open_timeout: 2, read_timeout: 2) do |http|
+        http.get(uri.path).body
+      end
+      JSON.parse(body)['prefixes'].map { IPAddr.new it.values.first }
+    end
   end
-  private_class_method :resolves?
 end

data/lib/drive/version.rb

@@ -1,4 +1,4 @@
 module Drive
   # Version of the gem, read by the gemspec.
-  VERSION = '0.4.0'.freeze
+  VERSION = '0.5.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: drive
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - claudiob