seo_geo 0.1.0

data/lib/seo_geo/audit/runner.rb

@@ -0,0 +1,327 @@
+# frozen_string_literal: true
+
+require "uri"
+require "rack"
+require "rack/mock"
+
+module SeoGeo
+  module Audit
+    # Requests the application in-process (Rack, no network, no server) for every known host and
+    # collects {Finding}s. Outside an indexable environment the run temporarily switches
+    # {Configuration#environment} to the first of +indexable_environments+, so a development
+    # checkout is audited as production would serve it.
+    #
+    # Per host: robots.txt (200), sitemap.xml (valid; 404 when the policy has none), every sitemap URL
+    # (up to +max_pages+) and "/", llms.txt and its links. Per HTML page: one +<title>+ (length, unique),
+    # description length, canonical origin, noindex vs sitemap, robots vs sitemap, JSON-LD parse and
+    # required properties, FAQPage text visible, +<html lang>+, Markdown alternate (+.md+ and
+    # +Accept: text/markdown+ with +Vary: Accept+), verification metas. Plus {StaticScan}.
+    #
+    # @example
+    #   report = SeoGeo::Audit::Runner.new(app: Rails.application, root: Rails.root).run
+    #   puts report
+    class Runner
+      MARKDOWN_TYPE = %r{\Atext/markdown}i
+
+      # @param app [#call] Rack application
+      # @param config [Configuration]
+      # @param sitemap [Sitemap::Registry]
+      # @param root [String, Pathname, nil] application root for {StaticScan}; nil skips it
+      # @param hosts [Array<String>, nil] hosts to audit; nil => {Configuration#known_hosts}
+      # @param max_pages [Integer] sitemap URLs fetched per host
+      def initialize(app:, config: SeoGeo.config, sitemap: SeoGeo.sitemap, root: nil, hosts: nil, max_pages: 200)
+        @app = app
+        @config = config
+        @sitemap = sitemap
+        @root = root
+        @hosts = hosts
+        @max_pages = max_pages
+        @report = Report.new
+        @titles = Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      # @return [Report]
+      def run
+        StaticScan.new(@root).call(@report) if @root
+        unmatched_index_collections
+        with_indexable_environment do
+          (@hosts || @config.known_hosts).each { |host| audit_host(host.to_s.downcase) }
+        end
+        duplicate_titles
+        @report
+      end
+
+      private
+
+      def audit_host(host)
+        policy = @config.host_policy(host)
+        robots = fetch(host, "/robots.txt")
+        evaluator = nil
+        if robots.status == 200
+          evaluator = Robots::Evaluator.new(robots.body)
+        else
+          error(host, "/robots.txt", "robots.status", "robots.txt returned #{robots.status}")
+        end
+
+        urls = policy.sitemap ? sitemap_urls(host) : expect_no_sitemap(host)
+        pages = urls.first(@max_pages)
+        @report.stats["skipped"] += urls.size - pages.size if urls.size > pages.size
+        pages.each do |url|
+          check_robots_allows(host, url, evaluator) if evaluator
+          audit_page(host, url, from_sitemap: true)
+        end
+        home = "#{@config.origin_for(host)}/"
+        unless pages.any? { |url| same_page?(url, home) }
+          audit_page(host, home, from_sitemap: false, full: policy.sitemap)
+        end
+        audit_llms(host) if policy.sitemap && @config.llms_txt.enabled
+      end
+
+      def sitemap_urls(host)
+        response = fetch(host, "/sitemap.xml")
+        unless response.status == 200
+          error(host, "/sitemap.xml", "sitemap.status", "sitemap.xml returned #{response.status} but the host policy has a sitemap")
+          return []
+        end
+
+        origin = "#{@config.origin_for(host)}/"
+        problems, locs, kind = Checks.sitemap(response.body, origin: origin)
+        problems.each { |message| error(host, "/sitemap.xml", "sitemap.valid", message) }
+        return locs unless kind == :sitemapindex
+
+        locs.flat_map do |part|
+          part_response = fetch(host, request_path(part))
+          part_problems, part_locs, = Checks.sitemap(part_response.body, origin: origin)
+          part_problems.each { |message| error(host, part, "sitemap.valid", message) }
+          part_locs
+        end
+      end
+
+      def expect_no_sitemap(host)
+        response = fetch(host, "/sitemap.xml")
+        if response.status == 200
+          error(host, "/sitemap.xml", "sitemap.unexpected", "sitemap.xml is served but the host policy says no sitemap")
+        end
+        []
+      end
+
+      def check_robots_allows(host, url, evaluator)
+        path = request_path(url)
+        # Groups the configuration denies on purpose (crawlers.training = :deny, overrides) are not problems.
+        agents = (["*"] + (evaluator.agents - ["*"])).uniq.reject { |agent| @config.crawlers.policy_for(agent) == :deny }
+        blocked = agents.reject { |agent| evaluator.allowed?(agent, path) }
+        return if blocked.empty?
+
+        error(host, url, "sitemap.robots_blocked", "listed in sitemap.xml but robots.txt disallows it for #{blocked.join(", ")}")
+      end
+
+      # +full: false+ (hosts without a sitemap, e.g. aliases): status, title count and canonical only.
+      def audit_page(host, url, from_sitemap:, full: true)
+        response = fetch(host, request_path(url))
+        @report.stats["pages"] += 1
+        unless response.status == 200
+          severity = from_sitemap || response.status >= 400 ? :error : :warning
+          @report.add(severity, "page.status", "returned #{response.status}#{location(response)}", host: host, url: url)
+          return
+        end
+        return unless response.content_type.to_s.include?("html")
+
+        doc = HtmlDocument.new(response.body)
+        return check_canonical(host, url, doc) unless full
+
+        noindex = doc.noindex?(response.headers["x-robots-tag"])
+        error(host, url, "sitemap.noindex", "listed in sitemap.xml but the page is noindex") if noindex && from_sitemap
+        check_title_and_description(host, url, doc, noindex)
+        check_canonical(host, url, doc)
+        Checks.json_ld(doc).each { |message| error(host, url, "json_ld", message) }
+        Checks.faq_visible(doc).each { |message| error(host, url, "faq.visible", message) }
+        warning(host, url, "html.lang", "<html> has no lang attribute") unless doc.html_lang
+        check_verification(host, url, doc) if same_page?(url, "#{@config.canonical_origin}/")
+        check_markdown(host, url, doc) if markdown_expected?(url, doc)
+      end
+
+      def check_canonical(host, url, doc)
+        Checks.title_count(doc).each { |message| error(host, url, "title.count", message) }
+        expected = URI.parse(@config.canonical_url(url))
+        Checks.canonical_origin(doc.canonical, "#{expected.scheme}://#{expected.host}#{port_suffix(expected)}").each do |message|
+          error(host, url, "canonical", message)
+        end
+      end
+
+      def check_title_and_description(host, url, doc, noindex)
+        title = doc.title
+        if title
+          warning(host, url, "title.length", "title is #{title.length} characters (> #{Checks::TITLE_MAX})") if title.length > Checks::TITLE_MAX
+          @titles[[host, title]] << url unless noindex
+        end
+        description = doc.description.to_s.strip
+        if description.empty?
+          warning(host, url, "description.missing", "no meta description")
+        elsif !Checks::DESCRIPTION_RANGE.cover?(description.length)
+          warning(host, url, "description.length",
+                  "description is #{description.length} characters (#{Checks::DESCRIPTION_RANGE.min}–#{Checks::DESCRIPTION_RANGE.max} expected)")
+        end
+      end
+
+      def check_verification(host, url, doc)
+        @config.verification.meta_tags.each do |name, value|
+          next if doc.meta_content(name) == value
+
+          error(host, url, "verification", "<meta name=\"#{name}\"> missing or different from the configured token")
+        end
+      end
+
+      # Pages registered with +markdown: false+ are not checked for Markdown (they do not serve it).
+      def markdown_expected?(url, doc)
+        @config.markdown.suffix_urls? && doc.markdown_alternate && !@sitemap.markdown_disabled?(request_path(url))
+      end
+
+      def check_markdown(host, url, doc)
+        markdown_url = doc.markdown_alternate
+        dot_md = fetch(host, request_path(markdown_url))
+        unless dot_md.status == 200 && dot_md.content_type.to_s.match?(MARKDOWN_TYPE)
+          error(host, markdown_url, "markdown.url", "advertised Markdown URL returned #{dot_md.status} #{dot_md.content_type}")
+        end
+
+        negotiated = fetch(host, request_path(url), "HTTP_ACCEPT" => "text/markdown")
+        unless negotiated.status == 200 && negotiated.content_type.to_s.match?(MARKDOWN_TYPE)
+          error(host, url, "markdown.accept", "page advertises Markdown but Accept: text/markdown returned " \
+                                              "#{negotiated.status} #{negotiated.content_type} (include SeoGeo::Markdownable " \
+                                              "or pass seo markdown: false)")
+          return
+        end
+        vary = negotiated.headers["vary"].to_s.split(",").map { |value| value.strip.downcase }
+        error(host, url, "markdown.vary", "Markdown response has no Vary: Accept") unless vary.include?("accept")
+      end
+
+      def audit_llms(host)
+        response = fetch(host, "/llms.txt")
+        unless response.status == 200
+          error(host, "/llms.txt", "llms.status", "llms.txt returned #{response.status}")
+          return
+        end
+
+        origins = [@config.origin_for(host), @config.canonical_origin].uniq
+        response.body.scan(/\]\((https?:\/\/[^)\s]+)\)/).flatten.uniq.each do |link|
+          next unless origins.any? { |origin| link.start_with?("#{origin}/") || link == origin }
+
+          target = URI.parse(link)
+          link_host = target.host.to_s.downcase
+          link_response = fetch(link_host, request_path(link))
+          @report.stats["llms_links"] += 1
+          unless link_response.status == 200
+            error(host, link, "llms.link", "llms.txt link returned #{link_response.status}#{location(link_response)}")
+            next
+          end
+          # .md links exist only for entries with markdown: true (or nil + markdown.enabled); they are served
+          # by SeoGeo::Markdown::PathSuffix + Markdownable. A 406 above means the action is not markdownable.
+          next unless target.path.to_s.end_with?(".md") && !link_response.content_type.to_s.match?(MARKDOWN_TYPE)
+
+          error(host, link, "llms.markdown", "llms.txt links the .md form but it returned #{link_response.content_type} " \
+                                             "(pass markdown: false to llms_txt.page/link for non-Markdownable actions)")
+        end
+      end
+
+      def duplicate_titles
+        @titles.each do |(host, title), urls|
+          next if urls.uniq.size < 2
+
+          warning(host, urls.first, "title.duplicate", "title #{title.inspect} is shared by #{urls.uniq.size} URLs: #{urls.uniq.first(5).join(", ")}")
+        end
+      end
+
+      # @return [Rack::MockResponse]
+      def fetch(host, path, headers = {})
+        origin = URI.parse(@config.origin_for(host))
+        scheme = origin.scheme || "https"
+        # Rails HostAuthorization (config.hosts) reads the Host header, not SERVER_NAME: send both, with the port
+        # only when it is not the scheme's default.
+        authority = origin.port && origin.port != origin.default_port ? "#{host}:#{origin.port}" : host
+        env = Rack::MockRequest.env_for("#{scheme}://#{authority}#{path}",
+                                        { "REMOTE_ADDR" => "127.0.0.1", "HTTP_ACCEPT" => "text/html" }.merge(headers))
+        env["HTTP_HOST"] = authority
+        env["HTTPS"] = "on" if scheme == "https"
+        status, response_headers, body = @app.call(env)
+        response = Rack::MockResponse.new(status, response_headers, body)
+        if status == 403 && host_blocked?(host, response)
+          error(host, path, "host_authorization", "Rails HostAuthorization blocked Host: #{authority}; run it through " \
+                                                  "`bin/rails seo_geo:audit` (or set SEO_GEO_AUDIT=1 before boot, e.g. for " \
+                                                  "bin/rails runner) so seo_geo allows its hosts outside production, or " \
+                                                  "add #{host.inspect} to config.hosts")
+        end
+        response
+      rescue StandardError => e
+        error(host, path, "exception", "#{e.class}: #{e.message.lines.first.to_s.strip}")
+        Rack::MockResponse.new(500, {}, [])
+      end
+
+      # Rails HostAuthorization answers 403 with an empty body unless detailed exceptions are on, so ask the
+      # app's own config.hosts as well as reading the "Blocked hosts" page.
+      def host_blocked?(host, response)
+        return true if response.body.include?("Blocked host")
+        return false unless @app.respond_to?(:config) && @app.config.respond_to?(:hosts) && !@app.config.hosts.empty?
+        return false unless defined?(ActionDispatch::HostAuthorization::Permissions)
+
+        !ActionDispatch::HostAuthorization::Permissions.new(@app.config.hosts).allows?(host)
+      end
+
+      def with_indexable_environment
+        return yield if @config.indexable_environment?
+
+        saved = @config.environment
+        begin
+          @config.environment = @config.indexable_environments.first.to_s
+          @report.stats["audited as #{@config.environment}"] = 1
+          yield
+        ensure
+          @config.environment = saved
+        end
+      end
+
+      def request_path(url)
+        uri = URI.parse(url)
+        path = uri.path.to_s.empty? ? "/" : uri.path
+        uri.query ? "#{path}?#{uri.query}" : path
+      rescue URI::InvalidURIError
+        url
+      end
+
+      def same_page?(url_a, url_b)
+        URI.parse(url_a).then { |a| [a.host, a.path.to_s.empty? ? "/" : a.path] } ==
+          URI.parse(url_b).then { |b| [b.host, b.path.to_s.empty? ? "/" : b.path] }
+      rescue URI::InvalidURIError
+        false
+      end
+
+      def port_suffix(uri)
+        [80, 443].include?(uri.port) ? "" : ":#{uri.port}"
+      end
+
+      def location(response)
+        target = response.headers["location"]
+        target ? " (→ #{target})" : ""
+      end
+
+      # Under default_noindex a collection added with index: true lists its records, but without match: the head
+      # cannot tell a record path belongs to it, so the page stays noindex unless the controller says
+      # +seo index: true+. The page audit then reports sitemap.noindex; this names the cause.
+      def unmatched_index_collections
+        return unless @config.default_noindex
+
+        @sitemap.unmatched_index_collections.each do |entry|
+          @report.add(:warning, "sitemap.collection_index",
+                      "collection #{entry.source.class} is added with index: true but no match:, so its record pages " \
+                      "are noindex unless their controller calls seo index: true (add match: ->(path) { ... })")
+        end
+      end
+
+      def error(host, url, check, message)
+        @report.add(:error, check, message, host: host, url: url)
+      end
+
+      def warning(host, url, check, message)
+        @report.add(:warning, check, message, host: host, url: url)
+      end
+    end
+  end
+end

data/lib/seo_geo/audit/static_scan.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  module Audit
+    # File-system checks that need no request: static files shadowing the engine routes, and app code
+    # that branches on the User-Agent header (a User-Agent is a claim, not an identity — anyone can send
+    # "GPTBot"; see {SeoGeo::BotVerifier}).
+    class StaticScan
+      SHADOWED_FILES = %w[robots.txt sitemap.xml llms.txt llms-full.txt].freeze
+      USER_AGENT_READ = /request\.user_agent|request\.headers\[\s*["']User-Agent["']\s*\]|HTTP_USER_AGENT/
+      BRANCHING = /\b(?:if|unless|elsif|when|case|while|until)\b|&&|\|\||=~|\?\s|\.(?:match\??|include\?|start_with\?|end_with\?|in\?)|\?\z/
+      SCANNED = %w[app/**/*.rb app/**/*.erb lib/**/*.rb lib/**/*.rake].freeze
+
+      # @param root [String, Pathname] application root
+      def initialize(root)
+        @root = root.to_s
+      end
+
+      # @param report [Report]
+      # @return [Report]
+      def call(report)
+        shadowed_files(report)
+        user_agent_branches(report)
+        report
+      end
+
+      private
+
+      def shadowed_files(report)
+        SHADOWED_FILES.each do |name|
+          path = File.join(@root, "public", name)
+          next unless File.file?(path)
+
+          report.add(:error, "static.shadowing", "public/#{name} is served before the seo_geo route; delete it",
+                     url: "public/#{name}")
+        end
+      end
+
+      def user_agent_branches(report)
+        SCANNED.flat_map { |pattern| Dir.glob(File.join(@root, pattern)) }.uniq.sort.each do |file|
+          File.foreach(file).with_index(1) do |line, number|
+            code = line.chomp.sub(/#(?!\{).*\z/, "")
+            next unless code.match?(USER_AGENT_READ) && code.match?(BRANCHING)
+
+            report.add(:warning, "code.user_agent_branch",
+                       "branches on the User-Agent header; never use it for access, content or rate-limit " \
+                       "decisions (it is spoofable) — use SeoGeo::BotVerifier when a bot must be trusted",
+                       url: "#{relative(file)}:#{number}")
+          end
+        rescue ArgumentError, Errno::ENOENT
+          next
+        end
+      end
+
+      def relative(file)
+        file.delete_prefix("#{@root}/")
+      end
+    end
+  end
+end

data/lib/seo_geo/audit.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # +rake seo_geo:audit+: requests the app in-process for every known host and reports
+  # problems that search engines and AI crawlers would hit. See {Audit::Runner}.
+  module Audit
+  end
+end
+
+require_relative "audit/report"
+require_relative "audit/html_document"
+require_relative "audit/checks"
+require_relative "audit/static_scan"
+require_relative "audit/runner"

data/lib/seo_geo/bot_verifier.rb

@@ -0,0 +1,278 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+require "json"
+require "net/http"
+require "resolv"
+require "uri"
+
+module SeoGeo
+  # Optional: checks whether a request that *claims* to be a crawler (by User-Agent) really comes from
+  # that operator. A User-Agent is a claim, not an identity — anyone can send "Googlebot".
+  #
+  # seo_geo itself never uses this (or the User-Agent) for access decisions: robots.txt is the only
+  # place the gem reads crawler names, Markdown is chosen by the Accept header, and every client gets
+  # the same public content. Use the verifier only when your app must grant a bot something extra
+  # (skip a rate limit, allow a paywalled preview), and then require +verified == true+.
+  #
+  # Methods (sources checked 2026-10-14):
+  # - Reverse DNS + forward confirmation (PTR host ends in an operator domain, and that host resolves
+  #   back to the same IP):
+  #   - Googlebot: googlebot.com, google.com —
+  #     https://developers.google.com/crawling/docs/crawlers-fetchers/verify-google-requests
+  #     (googleusercontent.com is deliberately NOT accepted: it also names customer Google Cloud hosts and is
+  #     documented only for user-triggered fetchers, not Googlebot)
+  #   - Bingbot: search.msn.com — https://www.bing.com/toolbox/verify-bingbot
+  #   - Applebot: applebot.apple.com — https://support.apple.com/en-us/119829
+  #   - Yeti (Naver): naver.com — NOT documented by Naver (searchadvisor.naver.com only covers robots.txt);
+  #     the suffix is common practice (PTR names like crawl.<ip>.web.naver.com). Treat as best effort.
+  # - Published IP ranges (+{"prefixes":[{"ipv4Prefix"|"ipv6Prefix": CIDR}]}+), cached for +ttl+ seconds;
+  #   when a refresh fails the old list is used for at most another +ttl+, then the result is +nil+:
+  #   - OpenAI GPTBot / OAI-SearchBot / ChatGPT-User — https://developers.openai.com/api/docs/bots
+  #     (openai.com/gptbot.json, searchbot.json, chatgpt-user.json)
+  #   - Perplexity PerplexityBot / Perplexity-User — https://docs.perplexity.ai/guides/bots
+  #   - Anthropic ClaudeBot / Claude-User / Claude-SearchBot — https://support.claude.com/en/articles/8896518
+  #     (one list for all three: claude.com/crawling/bots.json)
+  # - Everything else (CCBot, Meta, Bytespider, ...) => +verified: nil+ (no published method).
+  #
+  # Web Bot Auth (HTTP Message Signatures for bots, IETF draft-meunier-webbotauth-httpsig-protocol):
+  # requests carrying +Signature-Agent+, or +Signature+ with +Signature-Input+, are reported as
+  # +signed: true+. Verifying the signature is TODO — +signed+ says nothing about who signed.
+  #
+  # @example config/application.rb (opt-in; place after ActionDispatch::RemoteIp so proxies are honoured)
+  #   config.middleware.use SeoGeo::BotVerifier::Middleware
+  #   # later, in a controller:
+  #   bot = request.env["seo_geo.bot"]   # nil for ordinary visitors
+  #   skip_rate_limit if bot && bot[:verified] == true
+  class BotVerifier
+    ENV_KEY = "seo_geo.bot"
+    DEFAULT_TTL = 24 * 60 * 60
+    # Seconds a failed range download is not retried (avoids stacking timeouts behind the refresh lock).
+    FAILURE_BACKOFF = 60
+
+    OPENAI = "https://openai.com"
+    PERPLEXITY = "https://www.perplexity.com"
+    ANTHROPIC_RANGES = "https://claude.com/crawling/bots.json"
+
+    # Verification method per User-Agent token: +dns:+ PTR suffixes, or +ranges:+ IP-range JSON URL.
+    METHODS = {
+      "Googlebot" => { dns: %w[googlebot.com google.com] },
+      "Bingbot" => { dns: %w[search.msn.com] },
+      "Applebot" => { dns: %w[applebot.apple.com] },
+      "Yeti" => { dns: %w[naver.com] },
+      "GPTBot" => { ranges: "#{OPENAI}/gptbot.json" },
+      "OAI-SearchBot" => { ranges: "#{OPENAI}/searchbot.json" },
+      "ChatGPT-User" => { ranges: "#{OPENAI}/chatgpt-user.json" },
+      "PerplexityBot" => { ranges: "#{PERPLEXITY}/perplexitybot.json" },
+      "Perplexity-User" => { ranges: "#{PERPLEXITY}/perplexity-user.json" },
+      "ClaudeBot" => { ranges: ANTHROPIC_RANGES },
+      "Claude-User" => { ranges: ANTHROPIC_RANGES },
+      "Claude-SearchBot" => { ranges: ANTHROPIC_RANGES }
+    }.freeze
+
+    # Crawlers that fetch pages but are not in {Crawlers::ALL} (whose Apple entry is the robots-only
+    # Applebot-Extended token).
+    EXTRA_CRAWLERS = { "Applebot" => :search }.freeze
+
+    # Default DNS resolver: Resolv with a 2-second timeout. Anything with +getnames(ip)+ and
+    # +getaddresses(name)+ works.
+    def self.default_resolver
+      Resolv::DNS.new.tap { |dns| dns.timeouts = 2 }
+    end
+
+    # Default range fetcher: +(url) -> body+ over HTTPS with short timeouts; raises on non-200.
+    DEFAULT_FETCHER = lambda do |url|
+      uri = URI.parse(url)
+      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 3, read_timeout: 5) do |http|
+        http.get(uri.request_uri, "Accept" => "application/json", "User-Agent" => "seo_geo/#{SeoGeo::VERSION}")
+      end
+      raise IOError, "GET #{url} returned #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+      response.body
+    end
+
+    # @return [BotVerifier] process-wide instance used by {Middleware} when none is given
+    def self.default
+      @default ||= new
+    end
+
+    # @param resolver [#getnames, #getaddresses] DNS (tests inject a stub)
+    # @param fetcher [#call] +(url) -> String+ JSON body (tests inject a stub)
+    # @param ttl [Integer] seconds an IP-range list and a DNS answer stay cached
+    # @param clock [#call] monotonic seconds (tests)
+    def initialize(resolver: self.class.default_resolver, fetcher: DEFAULT_FETCHER, ttl: DEFAULT_TTL,
+                   clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+      @resolver = resolver
+      @fetcher = fetcher
+      @ttl = ttl
+      @clock = clock
+      @ranges = {}
+      @dns = {}
+      @mutex = Mutex.new
+      @refresh = Mutex.new
+    end
+
+    # @param user_agent [String, nil] User-Agent header
+    # @param ip [String, nil] client IP
+    # @param headers [Hash{String => String}] Rack env or header hash (for Web Bot Auth detection)
+    # @return [Hash, nil] +{name:, purpose:, claimed:, verified:, signed:}+; nil when the request neither
+    #   claims a known crawler nor carries Web Bot Auth headers. +verified+ is true/false, or nil when the
+    #   operator publishes no verification method or the check could not run (DNS/range fetch failure).
+    def verify(user_agent:, ip:, headers: {})
+      name, purpose = identify(user_agent)
+      signed = signed?(headers)
+      return nil if name.nil? && !signed
+
+      { name: name, purpose: purpose, claimed: !name.nil?, verified: name && verified?(name, ip), signed: signed }
+    end
+
+    # @param user_agent [String, nil]
+    # @return [Array(String, Symbol), nil] token and purpose of the crawler the User-Agent claims
+    def identify(user_agent)
+      return nil if user_agent.nil? || user_agent.strip.empty?
+
+      crawler = Crawlers.find(user_agent)
+      crawler = nil if crawler&.token_only
+      return [crawler.user_agent, crawler.purpose] if crawler
+
+      token = EXTRA_CRAWLERS.keys.find { |name| user_agent.downcase.include?(name.downcase) }
+      token && [token, EXTRA_CRAWLERS[token]]
+    end
+
+    # @param name [String] crawler token
+    # @param ip [String, nil]
+    # @return [Boolean, nil]
+    def verified?(name, ip)
+      method = METHODS[name]
+      address = parse_ip(ip)
+      return nil if method.nil?
+      return false if address.nil?
+
+      method[:dns] ? dns_verified?(address, method[:dns]) : in_ranges?(address, method[:ranges])
+    end
+
+    # @param headers [Hash]
+    # @return [Boolean] true when Web Bot Auth headers are present (signature NOT verified)
+    def signed?(headers)
+      lookup = ->(name) { headers[name] || headers["HTTP_#{name.upcase.tr("-", "_")}"] || headers[name.downcase] }
+      present = ->(value) { !value.nil? && !value.to_s.strip.empty? }
+      present.call(lookup.call("Signature-Agent")) ||
+        (present.call(lookup.call("Signature")) && present.call(lookup.call("Signature-Input")))
+    end
+
+    # Drops cached ranges and DNS answers.
+    # @return [void]
+    def clear_cache
+      @mutex.synchronize do
+        @ranges.clear
+        @dns.clear
+      end
+    end
+
+    private
+
+    # A single host address (no CIDR notation); IPv4-mapped IPv6 is folded to IPv4.
+    def parse_ip(ip)
+      text = ip.to_s.strip
+      return nil if text.empty? || text.include?("/")
+
+      address = IPAddr.new(text)
+      address.ipv4_mapped? ? address.native : address
+    rescue IPAddr::Error
+      nil
+    end
+
+    def dns_verified?(address, suffixes)
+      key = [address.to_s, suffixes]
+      cached(@dns, key) do
+        names = @resolver.getnames(address.to_s).map { |name| name.to_s.downcase.chomp(".") }
+        match = names.find { |name| suffixes.any? { |suffix| name == suffix || name.end_with?(".#{suffix}") } }
+        match ? @resolver.getaddresses(match).any? { |forward| same_ip?(forward, address) } : false
+      end
+    rescue Resolv::ResolvError, Resolv::ResolvTimeout, SocketError, SystemCallError
+      nil
+    end
+
+    def same_ip?(candidate, address)
+      IPAddr.new(candidate.to_s) == address
+    rescue IPAddr::Error
+      false
+    end
+
+    def in_ranges?(address, url)
+      prefixes = ranges(url)
+      prefixes && prefixes.any? { |range| range.family == address.family && range.include?(address) }
+    end
+
+    # Cached prefixes for +url+. Refreshes are serialized (a slow, older response can never overwrite a
+    # newer list). A failed refresh keeps the previous list for at most another +ttl+; nil means unknown.
+    def ranges(url)
+      entry = @mutex.synchronize { @ranges[url] }
+      return entry[:value] if entry && entry[:expires] > now
+
+      @refresh.synchronize do
+        entry = @mutex.synchronize { @ranges[url] }
+        return entry[:value] if entry && entry[:expires] > now
+
+        begin
+          value = parse_ranges(@fetcher.call(url))
+          @mutex.synchronize { @ranges[url] = { value: value, expires: now + @ttl, stale_until: now + (2 * @ttl) } }
+          value
+        rescue StandardError
+          stale = entry && entry[:stale_until] > now ? entry[:value] : nil
+          retry_at = now + [FAILURE_BACKOFF, @ttl].min
+          retry_at = [retry_at, entry[:stale_until]].min if stale
+          @mutex.synchronize do
+            @ranges[url] = { value: stale, expires: retry_at, stale_until: entry ? entry[:stale_until] : now }
+          end
+          stale
+        end
+      end
+    end
+
+    def parse_ranges(body)
+      JSON.parse(body.to_s).fetch("prefixes").filter_map do |prefix|
+        cidr = prefix["ipv4Prefix"] || prefix["ipv6Prefix"]
+        cidr && IPAddr.new(cidr)
+      end
+    end
+
+    def cached(store, key)
+      entry = @mutex.synchronize { store[key] }
+      return entry[:value] if entry && entry[:expires] > now
+
+      value = yield
+      @mutex.synchronize do
+        store.shift if store.size >= 10_000
+        store[key] = { value: value, expires: now + @ttl }
+      end
+      value
+    end
+
+    def now
+      @clock.call
+    end
+
+    # Rack middleware: sets +env["seo_geo.bot"]+ (see {BotVerifier#verify}) and nothing else — it never
+    # blocks, redirects or changes the response.
+    class Middleware
+      # @param app [#call]
+      # @param verifier [BotVerifier]
+      def initialize(app, verifier: BotVerifier.default)
+        @app = app
+        @verifier = verifier
+      end
+
+      def call(env)
+        env[ENV_KEY] = @verifier.verify(user_agent: env["HTTP_USER_AGENT"], ip: client_ip(env), headers: env)
+        @app.call(env)
+      end
+
+      private
+
+      def client_ip(env)
+        env["action_dispatch.remote_ip"]&.to_s || env["REMOTE_ADDR"]
+      end
+    end
+  end
+end