seo_geo 0.1.0

data/lib/seo_geo/og_image/card.html.erb

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="utf-8">
+<style>
+  html, body { margin: 0; width: <%= width %>px; height: <%= height %>px; }
+  body {
+    box-sizing: border-box; padding: 72px 84px; display: flex; flex-direction: column; justify-content: flex-end;
+    background: #f5f5f7; color: #111; font-family: -apple-system, "Segoe UI", "Noto Sans KR", "Apple SD Gothic Neo", sans-serif;
+    word-break: keep-all; overflow-wrap: break-word;
+  }
+  .site { position: absolute; top: 72px; left: 84px; font-size: 32px; font-weight: 600; color: #555; }
+  h1 { margin: 0; font-size: 72px; line-height: 1.15; text-wrap: balance; }
+  p { margin: 24px 0 0; font-size: 34px; line-height: 1.4; color: #555; text-wrap: pretty; }
+</style>
+</head>
+<body>
+  <% if site.name %><div class="site"><%= h(site.name) %></div><% end %>
+  <h1><%= h(page&.heading || page&.title || site.name || key) %></h1>
+  <% if page&.description %><p><%= h(page.description) %></p><% end %>
+</body>
+</html>

data/lib/seo_geo/og_image/ferrum_backend.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module SeoGeo
+  module OgImage
+    # Built-in backend: renders an ERB template to HTML and screenshots it with headless Chrome via
+    # the optional ferrum gem. Add +gem "ferrum"+ and a Chrome/Chromium binary to use it.
+    #
+    # Template locals: +key+, +page+ (or nil), +params+, +site+ (SiteOptions), +width+, +height+.
+    # Everything interpolated must go through +h(...)+ — the template is HTML.
+    class FerrumBackend
+      DEFAULT_TEMPLATE = File.expand_path("card.html.erb", __dir__)
+
+      # @param config [Configuration]
+      # @param browser_options [Hash] extra options for +Ferrum::Browser.new+
+      def initialize(config: SeoGeo.config, browser_options: {})
+        @config = config
+        @browser_options = browser_options
+      end
+
+      # @return [String] PNG bytes
+      def call(key, page, params)
+        require "ferrum"
+        width = @config.og_image.width
+        height = @config.og_image.height
+        browser = Ferrum::Browser.new({ headless: true, window_size: [width, height], timeout: 15 }
+                                        .merge(@browser_options))
+        begin
+          # window_size includes browser chrome; the viewport must be exactly width x height.
+          browser.resize(width: width, height: height)
+          browser.go_to("about:blank")
+          browser.main_frame.content = html(key, page, params)
+          wait_for_assets(browser)
+          browser.screenshot(format: "png", encoding: :binary)
+        ensure
+          browser.quit
+        end
+      end
+
+      # Web fonts and images referenced by the template; a slow asset must not fail the card.
+      def wait_for_assets(browser)
+        browser.network.wait_for_idle(timeout: 5)
+      rescue Ferrum::TimeoutError
+        nil
+      end
+
+      # @return [String] the rendered HTML card
+      def html(key, page, params)
+        Template.new(File.read(@config.og_image.template || DEFAULT_TEMPLATE)).render(
+          key: key, page: page, params: params, site: @config.site,
+          width: @config.og_image.width, height: @config.og_image.height
+        )
+      end
+
+      # Minimal ERB context exposing locals and +h+.
+      class Template
+        include ERB::Util
+
+        def initialize(source)
+          @erb = ERB.new(source, trim_mode: "-")
+        end
+
+        def render(locals)
+          context = binding
+          locals.each { |name, value| context.local_variable_set(name, value) }
+          @erb.result(context)
+        end
+      end
+    end
+  end
+end

data/lib/seo_geo/og_image/renderer.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module SeoGeo
+  # Open Graph images served at +GET /og/:key.png+.
+  #
+  # A backend is any callable +(key, page, params) -> String+ returning PNG bytes (a Satori
+  # sidecar, a vips composer, ...). Without one, the bundled {FerrumBackend} renders an ERB
+  # card when the optional ferrum gem is installed. Results are cached in +Rails.cache+ for
+  # +og_image.cache_ttl+ under a key that includes +og_image.template_version+; failures are never cached.
+  #
+  # Every cache miss costs a render (with ferrum, a Chrome launch), so the controller serves only
+  # {.resolve_key known keys} and {.validated_params constrained params}, and at most
+  # +og_image.max_concurrent+ renders run at once per process.
+  module OgImage
+    # Raised when no backend can render, or a backend returns something that is not a PNG.
+    class RenderError < StandardError; end
+    # Raised on a cache miss while +og_image.max_concurrent+ renders are already running.
+    class Busy < RenderError; end
+
+    SLOTS = Mutex.new
+    @active_renders = 0
+
+    PNG_SIGNATURE = "\x89PNG\r\n\x1A\n".b.freeze
+    KEY_FORMAT = /\A[a-zA-Z0-9_-]{1,100}\z/
+
+    module_function
+
+    # @param key [String]
+    # @param page [Page, nil] page copy for +key+ when one is defined
+    # @param params [Hash{String => String}] permitted query parameters
+    # @param config [Configuration]
+    # @param cache [#fetch, nil] cache store; nil disables caching
+    # @raise [RenderError]
+    # @return [String] PNG bytes (binary)
+    # @raise [Busy] when every render slot is taken
+    def render(key, page: nil, params: {}, config: SeoGeo.config, cache: default_cache)
+      produce = -> { with_render_slot(config.og_image.max_concurrent) { render_uncached(key, page, params, config) } }
+      return produce.call if cache.nil?
+
+      cache.fetch(cache_key(key, page, params, config), expires_in: config.og_image.cache_ttl) { produce.call }
+    end
+
+    # Route constraint for +GET /og/:key.png+. The engine is mounted at "/" before or after the app's routes,
+    # so a request it does not want must not match at all (a controller 404 does not cascade): +/og/alice.png+
+    # reaches the app's own +/og/:username.png+ unless OG images are enabled and +alice+ is a known key.
+    # The resolved value is kept in +env+ for the controller, so +key_resolver+ runs once per request.
+    #
+    # A +key_resolver+ that raises claims the request (+true+) and leaves the error in +env+: the controller then
+    # answers with the configured fallback (redirect to +og_image.fallback_url+, else 404) and logs once. Falling
+    # through instead would hand a resolver outage to whatever app route matches next — a 404/500 that preview bots
+    # cache, or the wrong image — and skip +fallback_url+ entirely.
+    module RouteConstraint
+      ENV_KEY = "seo_geo.og_key"
+      ERROR_KEY = "seo_geo.og_key_error"
+
+      module_function
+
+      # @param request [ActionDispatch::Request]
+      # @return [Boolean]
+      def matches?(request)
+        config = SeoGeo.config
+        key = request.path_parameters[:key].to_s
+        return false unless config.og_image.enabled? && KEY_FORMAT.match?(key)
+
+        resolved = OgImage.resolve_key(key, config)
+        request.env[ENV_KEY] = resolved if resolved
+        resolved ? true : false
+      rescue StandardError => e
+        request.env[ERROR_KEY] = e
+        true
+      end
+    end
+
+    # @param key [String]
+    # @return [Page, true, nil] truthy when +key+ is a {SeoGeo.pages} key, listed in +og_image.keys+ or
+    #   accepted by +og_image.key_resolver+ (whose {Page} is returned); nil => 404
+    def resolve_key(key, config)
+      return true if config.og_image.keys.any? { |name| name.to_s == key }
+      return true if SeoGeo.pages.keys.any? { |name| name.to_s == key }
+
+      config.og_image.key_resolver&.call(key) || nil
+    end
+
+    # @param query [Hash] request query parameters
+    # @return [Hash{String => String}, nil] permitted params that carry a constraint; nil when any value
+    #   is not a String, is longer than +og_image.max_param_length+ or fails its constraint
+    def validated_params(query, config)
+      options = config.og_image
+      constraints = options.param_constraints.transform_keys(&:to_s)
+      names = options.permitted_params.map(&:to_s).select { |name| constraints.key?(name) }
+      params = query.to_h.transform_keys(&:to_s).slice(*names)
+      valid = params.all? do |name, value|
+        value.is_a?(String) && value.length <= options.max_param_length && allowed_value?(constraints[name], value)
+      end
+      valid ? params : nil
+    end
+
+    # @return [Boolean] whole-value match for a Regexp, membership for an Array. A Regexp that exceeds
+    #   {SafeRegexp::TIMEOUT} rejects the value (=> 400) instead of holding the worker.
+    def allowed_value?(rule, value)
+      return rule.map(&:to_s).include?(value) if rule.is_a?(Array)
+
+      match = SafeRegexp.match(rule, value)
+      !match.nil? && match.begin(0).zero? && match.end(0) == value.length
+    rescue Regexp::TimeoutError, SafeRegexp::TooLong
+      false
+    end
+
+    # Runs the block in one of +limit+ process-wide slots; never waits.
+    # @raise [Busy] when all slots are taken
+    def with_render_slot(limit)
+      acquired = SLOTS.synchronize do
+        next false if OgImage.active_renders >= limit
+
+        OgImage.active_renders += 1
+        true
+      end
+      raise Busy, "#{limit} OG image renders already running" unless acquired
+
+      begin
+        yield
+      ensure
+        SLOTS.synchronize { OgImage.active_renders -= 1 }
+      end
+    end
+
+    class << self
+      # @return [Integer] renders running right now in this process
+      attr_accessor :active_renders
+    end
+
+    # @return [String] cache key covering template version, key, locale and permitted params
+    def cache_key(key, page, params, config)
+      digest = Digest::SHA256.hexdigest(JSON.generate(params.to_h.sort))[0, 16]
+      ["seo_geo/og", config.og_image.template_version, key, page&.locale, digest].compact.join("/")
+    end
+
+    # @return [#call, nil] the configured backend, else ferrum when available
+    def backend(config)
+      config.og_image.backend || (config.og_image.ferrum_available? ? FerrumBackend.new(config: config) : nil)
+    end
+
+    # @return [#fetch, nil] Rails.cache when Rails is loaded
+    def default_cache
+      defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : nil
+    end
+
+    def render_uncached(key, page, params, config)
+      renderer = backend(config)
+      raise RenderError, "no og_image.backend configured and the ferrum gem is not available" if renderer.nil?
+
+      bytes = renderer.call(key, page, params)
+      unless bytes.is_a?(String) && bytes.b.start_with?(PNG_SIGNATURE)
+        raise RenderError, "og_image backend returned #{bytes.class} instead of PNG bytes"
+      end
+
+      bytes.b
+    end
+  end
+end

data/lib/seo_geo/page.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module SeoGeo
+  # The single source of page copy. The same object feeds the crawler fallback HTML, JSON-LD
+  # (FAQPage must match visible text) and Inertia props, so the three can never drift.
+  #
+  # @!attribute key [Symbol]
+  # @!attribute locale [Symbol, nil]
+  # @!attribute title [String, nil]
+  # @!attribute description [String, nil]
+  # @!attribute heading [String, nil] visible H1
+  # @!attribute intro [String, Array<String>, nil] lead paragraph, or several ({#intro_paragraphs})
+  # @!attribute features [Array<Hash{Symbol => String}>] each +{title:, description:}+
+  # @!attribute faq [Array<Hash{Symbol => String}>] each +{question:, answer:}+
+  # @!attribute extra [Hash{Symbol => Object}] any other keys from the source, passed through to {#props}
+  Page = Data.define(:key, :locale, :title, :description, :heading, :intro, :features, :faq, :extra) do
+    # Builds a page from a loosely shaped hash (YAML, I18n or Ruby). Accepts +q+/+a+ as FAQ aliases
+    # and +name+/+body+ as feature aliases.
+    # @param key [Symbol, String]
+    # @param data [Hash]
+    # @param locale [Symbol, nil]
+    # @return [Page]
+    def self.from_hash(key, data, locale: nil)
+      data = deep_copy(deep_symbolize(data || {}), freeze: true)
+      known = %i[title description heading intro features faq]
+      new(key: key.to_sym, locale: locale&.to_sym,
+          title: data[:title], description: data[:description], heading: data[:heading], intro: data[:intro],
+          features: Array(data[:features]).map { |item| normalize_feature(item) }.freeze,
+          faq: Array(data[:faq]).map { |item| normalize_faq(item) }.freeze,
+          extra: data.except(*known).freeze)
+    end
+
+    def self.deep_symbolize(value)
+      case value
+      when Hash then value.to_h { |name, inner| [name.to_sym, deep_symbolize(inner)] }
+      when Array then value.map { |inner| deep_symbolize(inner) }
+      else value
+      end
+    end
+
+    # @api private
+    # Copies Hash/Array/String nodes so a Page never shares mutable state with its source or its callers.
+    # @param value [Object]
+    # @param freeze [Boolean] freeze every copied node (registry storage) or leave them mutable ({#props})
+    # @return [Object]
+    def self.deep_copy(value, freeze:)
+      copy = case value
+             when Hash then value.to_h { |name, inner| [deep_copy(name, freeze: freeze), deep_copy(inner, freeze: freeze)] }
+             when Array then value.map { |inner| deep_copy(inner, freeze: freeze) }
+             when String then value.dup
+             else return value
+             end
+      freeze ? copy.freeze : copy
+    end
+
+    def self.normalize_feature(item)
+      return { title: item.to_s, description: nil }.freeze unless item.is_a?(Hash)
+
+      { title: item[:title] || item[:name], description: item[:description] || item[:body] }.freeze
+    end
+
+    def self.normalize_faq(item)
+      raise ConfigurationError, "faq items must be hashes with question/answer" unless item.is_a?(Hash)
+
+      { question: item[:question] || item[:q], answer: item[:answer] || item[:a] }.freeze
+    end
+    private_class_method :deep_symbolize, :normalize_feature, :normalize_faq
+
+    # @return [Array<String>] +intro+ as paragraphs (a String is one paragraph; blanks dropped)
+    def intro_paragraphs
+      Array(intro).map(&:to_s).reject { |line| line.strip.empty? }
+    end
+
+    # Shape (all keys always present, Symbol keys; +extra+ keys merged at the top level):
+    #   { key: Symbol, locale: Symbol | nil, title: String | nil, description: String | nil, heading: String | nil,
+    #     intro: String | Array<String> | nil (as written in the source),
+    #     features: [{ title:, description: }], faq: [{ question:, answer: }], **extra }
+    # Sources may write FAQ items as +q+/+a+ and features as +name+/+body+; props always carry the canonical
+    # +question+/+answer+ and +title+/+description+.
+    # @return [Hash{Symbol => Object}] detached, mutable deep copy for Inertia props / JSON APIs — changing it
+    #   never reaches the registry or other requests
+    def props
+      Page.deep_copy({ key: key, locale: locale, title: title, description: description, heading: heading,
+                       intro: intro, features: features, faq: faq }.merge(extra), freeze: false)
+    end
+
+    # Markdown rendering of the copy, used for Markdown negotiation on client-rendered pages.
+    # @return [String]
+    def to_markdown
+      parts = []
+      parts << "# #{heading || title}" if heading || title
+      parts.concat(intro_paragraphs)
+      unless features.empty?
+        parts << "## Features"
+        parts << features.map { |item| item[:description] ? "- **#{item[:title]}**: #{item[:description]}" : "- #{item[:title]}" }.join("\n")
+      end
+      unless faq.empty?
+        parts << "## FAQ"
+        faq.each { |item| parts << "### #{item[:question]}\n\n#{item[:answer]}" }
+      end
+      "#{parts.join("\n\n")}\n"
+    end
+  end
+
+  # Loads {Page}s from three interchangeable sources, in this precedence: Ruby hash, I18n scope, YAML file.
+  #
+  # YAML files live at +config/seo_geo/pages/<key>.yml+. A file is either a flat attribute hash or a
+  # hash keyed by locale (+ko:+, +en:+), in which case the requested locale (or site.default_locale) is used.
+  #
+  # @example
+  #   SeoGeo.pages.register(:pricing, title: "Pricing", faq: [{ q: "Free?", a: "Yes" }])
+  #   SeoGeo.pages.register_i18n(:login, scope: "views.login.seo", faq: :numbered) # q1_q / q1_a ...
+  #   SeoGeo.page(:home).props
+  class PageRegistry
+    # @return [Array<String>] directories searched for +<key>.yml+
+    attr_accessor :paths
+
+    def initialize
+      @paths = []
+      @hashes = {}
+      @i18n = {}
+    end
+
+    # @param key [Symbol, String]
+    # @param data [Hash] page attributes, or a hash keyed by locale
+    # @return [void]
+    def register(key, data = nil, **attributes)
+      @hashes[key.to_sym] = Page.deep_copy(data || attributes, freeze: true)
+    end
+
+    # @param key [Symbol, String]
+    # @param scope [String] I18n scope holding title/description/heading/intro/features/faq
+    # @param faq [Symbol] :list (an array under +faq+) or :numbered (+q1_q+/+q1_a+, +q2_q+/... keys)
+    # @return [void]
+    def register_i18n(key, scope:, faq: :list)
+      raise ConfigurationError, "faq: must be :list or :numbered" unless %i[list numbered].include?(faq)
+
+      @i18n[key.to_sym] = { scope: scope, faq: faq }
+    end
+
+    # @param key [Symbol, String]
+    # @param locale [Symbol, String, nil] nil => I18n.locale when available, else site.default_locale
+    # @raise [KeyError] when no source defines the page
+    # @return [Page]
+    def fetch(key, locale: nil)
+      key = key.to_sym
+      locale = (locale || current_locale).to_sym
+      data = if @hashes.key?(key) then localized(@hashes[key], locale)
+             elsif @i18n.key?(key) then from_i18n(@i18n[key], locale)
+             else from_yaml(key, locale)
+             end
+      raise KeyError, "seo_geo page #{key.inspect} is not defined (looked in Ruby, I18n and #{paths.inspect})" if data.nil?
+
+      Page.from_hash(key, data, locale: locale)
+    end
+
+    # @return [Array<Symbol>] every key known to any source
+    def keys
+      yaml_keys = paths.flat_map { |dir| Dir[File.join(dir, "*.yml")].map { |file| File.basename(file, ".yml").to_sym } }
+      (@hashes.keys + @i18n.keys + yaml_keys).uniq
+    end
+
+    # @return [void]
+    def clear
+      @hashes.clear
+      @i18n.clear
+    end
+
+    private
+
+    # The request-locale rule ({Configuration#request_locale}) shared with +<html lang>+ and +og:locale+, so a
+    # default fetch ({SeoGeo.page}, +seo_geo_page+, the crawler fallback) picks the same copy as the head.
+    def current_locale
+      SeoGeo.config.request_locale(defined?(I18n) && I18n.respond_to?(:locale) ? I18n.locale : nil)
+    end
+
+    def localized(data, locale)
+      data = Page.send(:deep_symbolize, data)
+      return data unless data.keys.any? { |name| name.to_s.match?(/\A[a-z]{2}(-[A-Za-z]{2,4})?\z/) && data[name].is_a?(Hash) }
+
+      data[locale] || data[SeoGeo.config.site.default_locale.to_sym]
+    end
+
+    def from_yaml(key, locale)
+      file = paths.map { |dir| File.join(dir, "#{key}.yml") }.find { |path| File.file?(path) }
+      file && localized(YAML.safe_load_file(file, aliases: true) || {}, locale)
+    end
+
+    def from_i18n(source, locale)
+      raw = I18n.t(source[:scope], locale: locale, default: nil)
+      return nil unless raw.is_a?(Hash)
+      return raw if source[:faq] == :list
+
+      numbered = raw.keys.filter_map { |name| name.to_s[/\Aq(\d+)_q\z/, 1]&.to_i }.sort
+      raw.reject { |name, _| name.to_s.match?(/\Aq\d+_[qa]\z/) }
+         .merge(faq: numbered.map { |n| { question: raw[:"q#{n}_q"], answer: raw[:"q#{n}_a"] } })
+    end
+  end
+end

data/lib/seo_geo/path_match.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # Declaration matching shared by {Configuration#noindex_path?} and {Sitemap::Registry} (+noindex?+,
+  # +index?+, +excluded?+). Paths and String declarations are compared as {RobotsPath.path} (query dropped,
+  # RFC 9309 normalization), so +/%61dmin+, +/admin?x=1+ and +/admin+ are one path.
+  module PathMatch
+    module_function
+
+    # @param value [String] path or absolute URL, raw or percent-encoded
+    # @return [Array<String>] the normalized path and, when it differs, its fully decoded UTF-8 form
+    #   (the forms a declaration Regexp is matched against)
+    def variants(value)
+      [RobotsPath.path(value), RobotsPath.decoded(value)].compact.uniq
+    end
+
+    # @param patterns [Array<String, Regexp>] exact paths (normalized like the candidate) or regexps
+    # @param path [String]
+    # @return [Boolean] a Regexp that times out ({SafeRegexp}) counts as a match: every declaration
+    #   restricts publication, so failing safe means restricting
+    def any?(patterns, path)
+      return false if patterns.nil? || patterns.empty?
+
+      plain = RobotsPath.path(path)
+      candidates = nil
+      patterns.any? do |pattern|
+        if pattern.is_a?(Regexp)
+          candidates ||= variants(path)
+          candidates.any? { |candidate| SafeRegexp.match?(pattern, candidate, on_timeout: true) }
+        else
+          RobotsPath.path(pattern) == plain
+        end
+      end
+    end
+  end
+end

data/lib/seo_geo/publication.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # One answer to "may this path be published to crawlers on this host?". sitemap.xml, llms.txt,
+  # llms-full.txt and the head's robots meta all ask it, so they cannot disagree.
+  #
+  # A path is publishable on a host when all hold:
+  # - the host serves a sitemap ({HostPolicy#sitemap}: declared/canonical, indexable, indexable environment);
+  # - robots.txt does not disallow it for the host (RFC 9309 longest match over the host's
+  #   +allow+/+disallow+ prefixes, Allow wins a tie);
+  # - it is not excluded with {Sitemap::Registry#exclude};
+  # - it is not declared noindex ({Configuration#noindex_paths}, +sitemap.add path, noindex: true+);
+  # - under {Configuration#default_noindex}, it is declared indexable (+sitemap.add path, index: true+ —
+  #   the registry counterpart of a controller's +seo index: true+).
+  #
+  # Request-time +seo noindex: true+ is invisible here; +rake seo_geo:audit+ reports such pages
+  # (+sitemap.noindex+).
+  module Publication
+    module_function
+
+    # @param host [String] request host
+    # @param path [String] path or absolute URL; robots rules see the {RobotsPath.target} (query kept),
+    #   declarations the {RobotsPath.path}. A +.md+ URL is judged as the page it is the Markdown of — callers
+    #   pass {RobotsPath.html_path} for those.
+    # @param config [Configuration]
+    # @param sitemap [Sitemap::Registry]
+    # @param check_host [Boolean] false skips the host sitemap check (callers that already did it)
+    # @param declared_index [Boolean] true when the caller already knows the entry was added with
+    #   +index: true+ (sitemap.xml, whose Registry#each_url only yields such entries under default_noindex)
+    # @return [Boolean]
+    def publishable?(host, path, config: SeoGeo.config, sitemap: SeoGeo.sitemap, check_host: true, declared_index: false)
+      policy = config.host_policy(host)
+      return false if check_host && !policy.sitemap
+      return false if robots_blocked?(policy, path)
+      return false if sitemap.excluded?(path)
+      return false if declared_noindex?(path, config: config, sitemap: sitemap)
+
+      !config.default_noindex || declared_index || declared_index?(host, path, config: config, sitemap: sitemap)
+    end
+
+    # Head robots meta: a declared-noindex path is noindex even when the controller calls +seo index: true+.
+    # Declarations apply on every host (restricting is the safe side).
+    # @param path [String]
+    # @return [Boolean]
+    def declared_noindex?(path, config: SeoGeo.config, sitemap: SeoGeo.sitemap)
+      config.noindex_path?(path) || sitemap.noindex?(path)
+    end
+
+    # Head robots meta under +default_noindex+: an +index: true+ entry for this host covers the path — a static
+    # path, or a collection whose +match:+ accepts it ({Sitemap::Entry#covers?}). The same answer
+    # sitemap.xml and llms.txt use, so the head cannot be noindex for a listed page on one host and indexable
+    # on another.
+    # @param host [String] request host
+    # @param path [String]
+    # @return [Boolean]
+    def declared_index?(host, path, config: SeoGeo.config, sitemap: SeoGeo.sitemap)
+      sitemap.index?(path, host: host, config: config)
+    end
+
+    # RFC 9309 §2.2.2 over the host policy's lists, through the same matcher as {Robots::Evaluator}.
+    # @param policy [HostPolicy]
+    # @param path [String]
+    # @return [Boolean]
+    def robots_blocked?(policy, path)
+      RobotsPath.blocked?(policy.allow, policy.disallow, path)
+    end
+  end
+end

data/lib/seo_geo/robots/evaluator.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  class Robots
+    # Reads a robots.txt body and answers "may this crawler fetch this path?" the way RFC 9309 says a
+    # crawler decides. Used by the test helpers and
+    # +rake seo_geo:audit+; it is never used for access decisions at request time.
+    #
+    # - Group choice (§2.2.1): the groups naming the crawler's product token (case-insensitive) are merged;
+    #   only when none names it does the crawler fall back to the +*+ groups.
+    # - Rule choice (§2.2.2): the longest matching path wins; +Allow+ wins a tie; no matching rule => allowed.
+    # - Patterns: +*+ matches any sequence, a trailing +$+ anchors the end. An empty +Disallow:+ is ignored.
+    #   Rules and paths are compared in {RobotsPath.normalize}d form: +/%61dmin+ is +/admin+, but an encoded
+    #   reserved delimiter stays one (+/download%3Fpublic+ is not +/download?public+).
+    #
+    # @example
+    #   rules = SeoGeo::Robots::Evaluator.new(body)
+    #   rules.allowed?("GPTBot", "/admin")                       # => false
+    #   rules.allowed?("Mozilla/5.0 (compatible; GPTBot/1.2)", "/") # full User-Agent headers work too
+    class Evaluator
+      Group = Struct.new(:agents, :rules)
+
+      # @param body [String] robots.txt contents
+      def initialize(body)
+        @groups = []
+        current = nil
+        body.to_s.each_line do |raw|
+          line = raw.sub(/#.*/, "").strip
+          next if line.empty?
+
+          field, value = line.split(":", 2).map { |part| part.to_s.strip }
+          case field.downcase
+          when "user-agent"
+            current = nil if current && !current.rules.empty?
+            current ||= Group.new([], []).tap { |group| @groups << group }
+            current.agents << value.downcase
+          when "allow", "disallow"
+            current&.rules&.push([field.downcase.to_sym, value])
+          end
+        end
+      end
+
+      # @param user_agent [String] robots.txt product token ("GPTBot", "*") or a full User-Agent header
+      # @param path [String] path, optionally with a query string
+      # @return [Boolean]
+      def allowed?(user_agent, path)
+        rule = matching_rule(user_agent, path)
+        rule.nil? || rule.first == :allow
+      end
+
+      # @param user_agent [String]
+      # @param path [String]
+      # @return [Array(Symbol, String), nil] the deciding +[:allow | :disallow, pattern]+, nil when no rule matches
+      def matching_rule(user_agent, path)
+        RobotsPath.decide(rules_for(user_agent), path.to_s.empty? ? "/" : path.to_s)
+      end
+
+      # @return [Array<String>] every agent token that has a group (lower case)
+      def agents
+        @groups.flat_map(&:agents)
+      end
+
+      # @param user_agent [String]
+      # @return [String] the group token the crawler obeys ("*" when no group names it)
+      def group_token(user_agent)
+        token_for(user_agent.to_s.downcase) || "*"
+      end
+
+      private
+
+      def rules_for(user_agent)
+        token = group_token(user_agent)
+        @groups.select { |group| group.agents.include?(token) }.flat_map(&:rules)
+      end
+
+      # Exact token first; otherwise the longest group token contained in a full User-Agent header.
+      def token_for(needle)
+        named = agents.uniq - ["*"]
+        return needle if named.include?(needle)
+
+        named.select { |token| needle.include?(token) }.max_by(&:length)
+      end
+    end
+  end
+end