seo_geo 0.1.0

data/lib/seo_geo/sitemap/registry.rb

@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  module Sitemap
+    CHANGEFREQS = %w[always hourly daily weekly monthly yearly never].freeze
+
+    # One resolved +<url>+ entry. +loc+ is absolute; +lastmod+ is nil unless a real date was supplied
+    # (seo_geo never invents +Time.now+).
+    Url = Data.define(:loc, :lastmod, :priority, :changefreq)
+
+    # One registered source: a static path or a record collection.
+    #
+    # @!attribute source [String, #call, #find_each, #each] path ("/about"), a lambda returning a scope (evaluated per
+    #   request: +-> { Post.published }+ — the recommended form, no model is loaded at boot and dev reloads never
+    #   leave a stale class behind) or a scope / enumerable of records
+    # @!attribute batch [Boolean, nil] nil/true => +find_each+ when the scope has it (batches by primary key:
+    #   +select+ must include +id+, +order+ is ignored); false => +each+, so the scope's own +order+ is kept
+    # @!attribute path [#call, Symbol, nil] record -> path; required for collections
+    # @!attribute lastmod [#call, Symbol, Time, Date, nil] date, record method name, or record -> date
+    # @!attribute priority [Float, nil] 0.0..1.0
+    # @!attribute changefreq [String, nil] one of {CHANGEFREQS}
+    # @!attribute hosts [Array<String>, nil] hosts whose sitemap lists the entry; nil => canonical host only
+    # @!attribute noindex [Boolean] declared noindex (static paths only): never listed in sitemap.xml or
+    #   llms.txt, and the head helper renders robots noindex for it
+    # @!attribute index [Boolean] declared indexable under {Configuration#default_noindex}; without it a
+    #   +default_noindex+ site lists nothing (see {SeoGeo::Publication})
+    # @!attribute match [Regexp, #call, nil] collections only: which request paths belong to the collection
+    #   (+->(path) { ... }+ or a Regexp over the path). With +index: true+ it is what makes the head of a record
+    #   page indexable under +default_noindex+ ({SeoGeo::Publication.declared_index?}); without it the head
+    #   needs +seo index: true+ and +rake seo_geo:audit+ warns (+sitemap.collection_index+)
+    # @!attribute markdown [Boolean, nil] true when the page also serves Markdown at +/path.md+ (its action includes
+    #   {SeoGeo::Markdownable}). The llms.txt sitemap fallback links +.md+ only for true; nil/false => HTML URL.
+    #   false also makes +rake seo_geo:audit+ skip the page's Markdown checks.
+    Entry = Data.define(:source, :path, :lastmod, :priority, :changefreq, :hosts, :noindex, :index, :markdown, :match,
+                       :batch) do
+      # @param config [Configuration]
+      # @return [Boolean]
+      def markdown?(config)
+        config.markdown.suffix_urls? && markdown == true
+      end
+
+      # @return [Boolean]
+      def static?
+        source.is_a?(String)
+      end
+
+      # @param host [String] request host
+      # @param config [Configuration]
+      # @return [Boolean]
+      def for_host?(host, config)
+        wanted = hosts ? hosts.map { |name| name.to_s.downcase } : [config.canonical_host]
+        wanted.include?(host.to_s.downcase.sub(/:\d+\z/, ""))
+      end
+
+      # @param path [String] request path
+      # @return [Boolean] true when +path+ is this entry's static path or matches its collection +match:+
+      #   (a timed-out Regexp does not match: membership grants indexing, so failing safe means not granting it)
+      def covers?(path)
+        return RobotsPath.path(source) == RobotsPath.path(path) if static?
+        return false if match.nil?
+        return SafeRegexp.match?(match, RobotsPath.path(path), on_timeout: false) if match.is_a?(Regexp)
+
+        match.call(RobotsPath.path(path)) ? true : false
+      end
+    end
+
+    # Boot-time registry of sitemap sources. Collections are stored unevaluated and enumerated per request.
+    #
+    # @example config/initializers/seo_geo.rb
+    #   SeoGeo.sitemap.draw do
+    #     add "/", priority: 1.0, changefreq: "daily"
+    #     add -> { Presentation.published }, path: ->(p) { "/p/#{p.slug}" }, lastmod: :updated_at
+    #     exclude %r{\A/s/}
+    #   end
+    class Registry
+      # @return [Array<Entry>]
+      attr_reader :entries
+      # @return [Array<String, Regexp>]
+      attr_reader :exclusions
+
+      def initialize
+        @entries = []
+        @exclusions = []
+      end
+
+      # Evaluates a block with this registry as +self+.
+      # @return [self]
+      def draw(&block)
+        instance_eval(&block) if block
+        self
+      end
+
+      # Registers a static path or a record collection.
+      # @param path_or_scope [String, #find_each, #each]
+      # @param path [#call, Symbol, nil] record -> path (collections only)
+      # @param lastmod [#call, Symbol, Time, Date, nil]
+      # @param priority [Numeric, nil] 0.0..1.0
+      # @param changefreq [String, Symbol, nil] one of {CHANGEFREQS}
+      # @param hosts [Array<String>, String, nil] nil => canonical host only
+      # @param noindex [Boolean] declare a static path noindex: it stays out of sitemap.xml and llms.txt and
+      #   the head helper renders robots noindex for it (see also {Configuration#noindex_paths})
+      # @param index [Boolean] declare the entry indexable when {Configuration#default_noindex} is on
+      #   (the registry counterpart of a controller's +seo index: true+)
+      # @param markdown [Boolean, nil] true => the llms.txt fallback links +/path.md+ (action includes
+      #   {SeoGeo::Markdownable}); false => also skip the audit's Markdown checks; nil => HTML URL
+      # @param match [Regexp, #call, nil] collections only: Regexp or +->(path) { bool }+ telling which request
+      #   paths are records of this collection, so +index: true+ reaches their head too
+      # @param batch [Boolean, nil] collections only: false iterates with +each+ (keeps +order+, loads all records);
+      #   default uses +find_each+ when available (+select+ must include +id+, +order+ is ignored)
+      # @raise [ConfigurationError] on invalid arguments
+      # @return [Entry]
+      def add(path_or_scope, path: nil, lastmod: nil, priority: nil, changefreq: nil, hosts: nil, noindex: false, index: false,
+              markdown: nil, match: nil, batch: nil)
+        raise ConfigurationError, "noindex: is only valid for static paths" if noindex && !path_or_scope.is_a?(String)
+        raise ConfigurationError, "match: is only valid for collections" if match && path_or_scope.is_a?(String)
+        raise ConfigurationError, "batch: is only valid for collections" if !batch.nil? && path_or_scope.is_a?(String)
+        if match && !(match.is_a?(Regexp) || match.respond_to?(:call))
+          raise ConfigurationError, "match: must be a Regexp or respond to #call"
+        end
+        raise ConfigurationError, "noindex: and index: are mutually exclusive" if noindex && index
+
+        if path_or_scope.is_a?(String)
+          raise ConfigurationError, "sitemap path #{path_or_scope.inspect} must start with /" unless path_or_scope.start_with?("/")
+          raise ConfigurationError, "path: is only valid for collections" if path
+        else
+          unless path_or_scope.respond_to?(:call) || path_or_scope.respond_to?(:find_each) || path_or_scope.respond_to?(:each)
+            raise ConfigurationError, "sitemap source must be a path String, a lambda returning a scope, or respond to #find_each/#each"
+          end
+          raise ConfigurationError, "collections need path: (a callable or a record method name)" unless path
+        end
+        if priority && !(priority.is_a?(Numeric) && priority.between?(0, 1))
+          raise ConfigurationError, "sitemap priority must be between 0.0 and 1.0 (got #{priority.inspect})"
+        end
+        if changefreq && !CHANGEFREQS.include?(changefreq.to_s)
+          raise ConfigurationError, "sitemap changefreq must be one of #{CHANGEFREQS.join(", ")}"
+        end
+
+        entry = Entry.new(source: path_or_scope, path: path, lastmod: lastmod, priority: priority&.to_f,
+                          changefreq: changefreq&.to_s, hosts: hosts && Array(hosts).map(&:to_s), noindex: noindex == true,
+                          index: index == true, markdown: markdown.nil? ? nil : markdown == true, match: match,
+                          batch: batch)
+        @entries << entry
+        entry
+      end
+
+      # Excludes paths from every sitemap (user-generated/share pages, duplicates).
+      # @param patterns [Array<String, Regexp>] exact paths or regexps matched against the path
+      # @return [void]
+      def exclude(*patterns)
+        @exclusions.concat(patterns.flatten)
+      end
+
+      # @param path [String]
+      # @return [Boolean] true when a static entry for +path+ was added with +noindex: true+
+      def noindex?(path)
+        PathMatch.any?(@entries.select(&:noindex).map(&:source), path)
+      end
+
+      # @param path [String]
+      # @param host [String, nil] request host; entries added with +hosts:+ count only there (nil => any host)
+      # @param config [Configuration] resolves +hosts: nil+ to the canonical host
+      # @return [Boolean] true when an entry added with +index: true+ covers +path+ on +host+: a static path, or
+      #   a collection whose +match:+ accepts it
+      def index?(path, host: nil, config: SeoGeo.config)
+        @entries.any? { |entry| entry.index && (host.nil? || entry.for_host?(host, config)) && entry.covers?(path) }
+      end
+
+      # @return [Array<Entry>] collections added with +index: true+ but no +match:+ — their record pages are
+      #   in sitemap.xml while the head cannot know they are declared indexable
+      def unmatched_index_collections
+        @entries.select { |entry| entry.index && !entry.static? && entry.match.nil? }
+      end
+
+      # @param path [String]
+      # @return [Boolean] true when a static entry for +path+ was added with +markdown: false+
+      def markdown_disabled?(path)
+        PathMatch.any?(@entries.select { |entry| entry.static? && entry.markdown == false }.map(&:source), path)
+      end
+
+      # @param path [String] compared like every declaration ({PathMatch}); a Regexp that times out excludes
+      # @return [Boolean]
+      def excluded?(path)
+        PathMatch.any?(@exclusions, path)
+      end
+
+      # Enumerates resolved URLs for one host. The sitemaps controller streams these into XML,
+      # splitting into a sitemap index above 50,000 URLs.
+      # @param host [String] request host
+      # @param config [Configuration]
+      # @yieldparam url [Url]
+      # @return [Enumerator<Url>] when no block is given
+      def each_url(host, config: SeoGeo.config)
+        return enum_for(:each_url, host, config: config) unless block_given?
+
+        origin = config.origin_for(host)
+        entries.each do |entry|
+          next if entry.noindex || !entry.for_host?(host, config)
+          next if config.default_noindex && !entry.index
+
+          each_record(entry) do |record|
+            relative = record ? resolve(entry.path, record) : entry.source
+            next if relative.nil? || excluded?(relative.to_s) || config.noindex_path?(relative.to_s)
+
+            lastmod = entry.lastmod.is_a?(Symbol) || entry.lastmod.respond_to?(:call) ? resolve(entry.lastmod, record) : entry.lastmod
+            yield Url.new(loc: config.canonical_url("#{origin}#{relative}"), lastmod: lastmod,
+                          priority: entry.priority, changefreq: entry.changefreq)
+          end
+        end
+      end
+
+      # Removes every entry and exclusion (tests, code reloading).
+      # @return [void]
+      def clear
+        @entries.clear
+        @exclusions.clear
+      end
+
+      private
+
+      def each_record(entry, &block)
+        return yield(nil) if entry.static?
+
+        source = entry.source.respond_to?(:call) ? entry.source.call : entry.source
+        if entry.batch != false && source.respond_to?(:find_each)
+          source.find_each(&block)
+        else
+          source.each(&block)
+        end
+      end
+
+      def resolve(callable_or_name, record)
+        return nil if callable_or_name.nil? || record.nil?
+        return record.public_send(callable_or_name) if callable_or_name.is_a?(Symbol)
+
+        callable_or_name.call(record)
+      end
+    end
+  end
+end

data/lib/seo_geo/sitemap/xml.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "date"
+require "uri"
+
+module SeoGeo
+  module Sitemap
+    # Renders the sitemap for one request host from {Registry#each_url}.
+    #
+    # Up to {MAX_URLS} URLs => one +<urlset>+ at /sitemap.xml. Above that, /sitemap.xml is a
+    # +<sitemapindex>+ pointing at /sitemap-1.xml, /sitemap-2.xml, ... (sitemaps.org limit: 50,000 per file).
+    # URLs {SeoGeo::Publication} rejects are dropped (robots-disallowed, excluded, declared noindex,
+    # not opted in under +default_noindex+).
+    #
+    # @example
+    #   xml = SeoGeo::Sitemap::Xml.new(SeoGeo.config, SeoGeo.sitemap, "www.example.com")
+    #   xml.render(nil) # => String, or nil when the host has no sitemap
+    class Xml
+      MAX_URLS = 50_000
+      XMLNS = "http://www.sitemaps.org/schemas/sitemap/0.9"
+
+      # @param config [Configuration]
+      # @param registry [Registry]
+      # @param host [String] request host
+      # @param max_urls [Integer] URLs per file (tests may lower it)
+      def initialize(config, registry, host, max_urls: MAX_URLS)
+        @config = config
+        @registry = registry
+        @host = host
+        @max_urls = max_urls
+        @policy = config.host_policy(host)
+      end
+
+      # @param page [Integer, nil] nil => /sitemap.xml; n => /sitemap-n.xml
+      # @return [String, nil] XML, or nil when there is nothing to serve (=> 404)
+      def render(page = nil)
+        return nil unless @policy.sitemap
+
+        page.nil? ? render_root : render_part(page)
+      end
+
+      # @return [Enumerator<Url>] URLs this host's sitemap lists
+      def urls
+        Enumerator.new do |yielder|
+          @registry.each_url(@host, config: @config) { |url| yielder << url if listable?(url) }
+        end
+      end
+
+      private
+
+      def render_root
+        head = urls.take(@max_urls + 1)
+        return nil if head.empty?
+        return urlset(head) if head.length <= @max_urls
+
+        sitemap_index
+      end
+
+      def render_part(page)
+        return nil unless page.is_a?(Integer) && page.positive?
+
+        chunk = []
+        split = false
+        urls.each_with_index do |url, index|
+          split = true if index == @max_urls
+          next unless index / @max_urls == page - 1
+          chunk << url
+        end
+        # Parts exist only when the root is an index; /sitemap-1.xml of a small site is a 404.
+        split && !chunk.empty? ? urlset(chunk) : nil
+      end
+
+      def sitemap_index
+        parts = []
+        urls.each_slice(@max_urls) { |slice| parts << slice.filter_map { |url| url.lastmod && as_time(url.lastmod) }.max }
+        origin = @config.origin_for(@host)
+        body = parts.each_with_index.map do |lastmod, index|
+          lines = ["  <sitemap>", "    <loc>#{escape("#{origin}/sitemap-#{index + 1}.xml")}</loc>"]
+          lines << "    <lastmod>#{format_time(lastmod)}</lastmod>" if lastmod
+          lines << "  </sitemap>"
+          lines.join("\n")
+        end
+        document("sitemapindex", body)
+      end
+
+      def urlset(list)
+        body = list.map do |url|
+          lines = ["  <url>", "    <loc>#{escape(url.loc)}</loc>"]
+          lines << "    <lastmod>#{format_time(url.lastmod)}</lastmod>" if url.lastmod
+          lines << "    <changefreq>#{url.changefreq}</changefreq>" if url.changefreq
+          lines << "    <priority>#{format("%.1f", url.priority)}</priority>" if url.priority
+          lines << "  </url>"
+          lines.join("\n")
+        end
+        document("urlset", body)
+      end
+
+      def document(root, body)
+        [%(<?xml version="1.0" encoding="UTF-8"?>), %(<#{root} xmlns="#{XMLNS}">), *body, "</#{root}>", ""].join("\n")
+      end
+
+      # {SeoGeo::Publication} decides (robots disallow, exclusions, noindex declarations,
+      # default_noindex) — the same predicate llms.txt and the head use.
+      def listable?(url)
+        # Registry#each_url already skipped entries without index: true under default_noindex.
+        Publication.publishable?(@host, url.loc, config: @config, sitemap: @registry, check_host: false, declared_index: true)
+      end
+
+      # Dates stay dates (W3C "YYYY-MM-DD"); times become W3C datetimes.
+      def format_time(value)
+        return value.to_s if value.is_a?(String)
+        return value.iso8601 if value.is_a?(Date) && !value.is_a?(DateTime)
+
+        value.respond_to?(:iso8601) ? value.iso8601 : value.to_s
+      end
+
+      def as_time(value)
+        case value
+        when DateTime then value.to_time
+        when Date then Time.utc(value.year, value.month, value.day)
+        when String then parse_time(value)
+        else value.respond_to?(:to_time) ? value.to_time : nil
+        end
+      end
+
+      def parse_time(value)
+        Time.iso8601(value)
+      rescue ArgumentError
+        date = Date.iso8601(value) rescue nil # rubocop:disable Style/RescueModifier
+        date && Time.utc(date.year, date.month, date.day)
+      end
+
+      def escape(text)
+        text.to_s.encode(xml: :text)
+      end
+    end
+  end
+end

data/lib/seo_geo/testing/environment.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  module Testing
+    # Runs the block as if the app were in another environment ({Configuration#environment}), then restores it,
+    # so a test can see production robots.txt, indexable heads and sitemaps. Shared by the Minitest assertions
+    # ({Assertions#with_seo_geo_environment}) and the RSpec helpers.
+    #
+    # @example
+    #   SeoGeo::Testing.with_environment("production") { get "/robots.txt" }
+    # @param name [String, Symbol]
+    # @param config [Configuration]
+    # @return [Object] the block's value
+    def self.with_environment(name, config: SeoGeo.config)
+      saved = config.environment
+      config.environment = name.to_s
+      yield
+    ensure
+      config.environment = saved
+    end
+
+    # Instance helper mixed into Minitest tests and RSpec examples.
+    module EnvironmentHelper
+      # @param name [String, Symbol]
+      # @return [Object] the block's value
+      def with_seo_geo_environment(name, &block)
+        Testing.with_environment(name, &block)
+      end
+    end
+  end
+end

data/lib/seo_geo/testing.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "seo_geo"
+require "uri"
+require_relative "testing/environment"
+
+module SeoGeo
+  # Minitest assertions for an app's integration tests. They use the same checks as
+  # +rake seo_geo:audit+ ({Audit::Checks}), so a passing test and a clean audit agree.
+  #
+  # @example test/integration/seo_test.rb
+  #   require "seo_geo/testing"
+  #
+  #   class SeoTest < ActionDispatch::IntegrationTest
+  #     include SeoGeo::Testing::Assertions
+  #
+  #     test("alias host folds")  { assert_seo_geo_canonical "/pricing", host: "example.com" }
+  #     test("faq is visible")    { assert_seo_geo_faq_matches_visible "/" }
+  #     test("one title")         { assert_seo_geo_single_title "/" }
+  #     test("sitemap")           { assert_seo_geo_sitemap host: "www.example.com" }
+  #     test("no alias sitemap")  { assert_seo_geo_no_sitemap host: "example.com" }
+  #     test("admin is closed")   { refute_seo_geo_robots_allows "GPTBot", "/admin", host: "www.example.com" }
+  #     test("production robots") { with_seo_geo_environment("production") { assert_seo_geo_sitemap host: "www.example.com" } }
+  #   end
+  module Testing
+    # Include into +ActionDispatch::IntegrationTest+ (needs +host!+, +get+ and +response+).
+    # Each assertion takes a +path+ and requests it; pass +path = nil+ to check the last response.
+    module Assertions
+      include EnvironmentHelper
+      # The page's +<link rel="canonical">+ equals +expected+ (default: {Configuration#canonical_url} of the
+      # requested URL — on an alias host that is the canonical origin).
+      # @param path [String, nil]
+      # @param host [String, nil]
+      # @param expected [String, nil]
+      def assert_seo_geo_canonical(path, host: nil, expected: nil)
+        seo_geo_get(path, host)
+        expected ||= SeoGeo.config.canonical_url(request.url)
+        actual = Audit::HtmlDocument.new(response.body).canonical
+        assert_equal expected, actual, "canonical of #{request.url}"
+      end
+
+      # Exactly one +<title>+ in the head.
+      # @param path [String, nil]
+      # @param host [String, nil]
+      def assert_seo_geo_single_title(path = nil, host: nil)
+        seo_geo_get(path, host)
+        problems = Audit::Checks.title_count(Audit::HtmlDocument.new(response.body))
+        assert_empty problems, "#{request.url}: #{problems.join("; ")}"
+      end
+
+      # Every FAQPage question and answer in the JSON-LD also appears in the visible text (or the
+      # crawler fallback body), and at least one FAQPage exists.
+      # @param path [String, nil]
+      # @param host [String, nil]
+      def assert_seo_geo_faq_matches_visible(path = nil, host: nil)
+        seo_geo_get(path, host)
+        doc = Audit::HtmlDocument.new(response.body)
+        refute_empty doc.faq_items, "#{request.url} has no FAQPage JSON-LD"
+        problems = Audit::Checks.faq_visible(doc)
+        assert_empty problems, "#{request.url}: #{problems.join("; ")}"
+      end
+
+      # /sitemap.xml on +host+ is 200, well-formed, in the sitemaps.org namespace, and lists only URLs
+      # on the host's origin.
+      # @param host [String]
+      # @return [Array<String>] the +<loc>+ values
+      def assert_seo_geo_sitemap(host:)
+        seo_geo_get("/sitemap.xml", host)
+        assert_equal 200, response.status, "/sitemap.xml on #{host}"
+        problems, locs, = Audit::Checks.sitemap(response.body, origin: "#{SeoGeo.config.origin_for(host)}/")
+        assert_empty problems, "/sitemap.xml on #{host}: #{problems.join("; ")}"
+        locs
+      end
+
+      # /sitemap.xml on +host+ is a 404 (alias, undeclared or non-indexable host).
+      # @param host [String]
+      def assert_seo_geo_no_sitemap(host:)
+        seo_geo_get("/sitemap.xml", host)
+        assert_equal 404, response.status, "/sitemap.xml on #{host} should be 404"
+      end
+
+      # robots.txt on +host+ lets +user_agent+ fetch +path+ (RFC 9309 evaluation, {Robots::Evaluator}).
+      # @param user_agent [String] token or full User-Agent header
+      # @param path [String]
+      # @param host [String, nil]
+      def assert_seo_geo_robots_allows(user_agent, path, host: nil)
+        assert seo_geo_robots(host).allowed?(user_agent, path), "robots.txt should allow #{user_agent} #{path}"
+      end
+
+      # robots.txt on +host+ disallows +path+ for +user_agent+.
+      def refute_seo_geo_robots_allows(user_agent, path, host: nil)
+        refute seo_geo_robots(host).allowed?(user_agent, path), "robots.txt should disallow #{user_agent} #{path}"
+      end
+
+      private
+
+      def seo_geo_robots(host)
+        seo_geo_get("/robots.txt", host)
+        assert_equal 200, response.status, "/robots.txt"
+        Robots::Evaluator.new(response.body)
+      end
+
+      def seo_geo_get(path, host)
+        return if path.nil?
+
+        host!(host) if host
+        get path
+      end
+    end
+  end
+end

data/lib/seo_geo/undeclared_host.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "set"
+
+module SeoGeo
+  # Fail-closed hosts are silent by design (noindex + +Disallow: /+), which also hides a misconfiguration: a proxy
+  # that drops the Host header (+127.0.0.1+), or an apex/www mismatch, deindexes the whole site. In an indexable
+  # environment the robots.txt controller and the head warn once per host per process.
+  module UndeclaredHost
+    MESSAGE = 'seo_geo: host %<host>s is not declared (canonical_origin/canonical_aliases/host); ' \
+              "serving noindex + Disallow: /. Check proxy Host headers."
+
+    @warned = Set.new
+    @lock = Mutex.new
+
+    module_function
+
+    # @param host [String, nil] request host
+    # @param config [Configuration]
+    # @return [Boolean] true when this call logged the warning
+    def warn_once(host, config: SeoGeo.config)
+      return false unless config.indexable_environment?
+      return false if config.host_policy(host.to_s).known
+
+      key = host.to_s.downcase
+      return false unless @lock.synchronize { @warned.add?(key) }
+
+      message = format(MESSAGE, host: key.inspect)
+      logger = defined?(Rails) && Rails.respond_to?(:logger) ? Rails.logger : nil
+      logger ? logger.warn(message) : Kernel.warn(message)
+      true
+    end
+
+    # Forgets which hosts were reported (tests).
+    # @return [void]
+    def reset!
+      @lock.synchronize { @warned.clear }
+    end
+  end
+end

data/lib/seo_geo/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  VERSION = "0.1.0"
+end

data/lib/seo_geo.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require_relative "seo_geo/version"
+require_relative "seo_geo/crawlers"
+require_relative "seo_geo/safe_regexp"
+require_relative "seo_geo/robots_path"
+require_relative "seo_geo/robots_directives"
+require_relative "seo_geo/undeclared_host"
+require_relative "seo_geo/configuration"
+require_relative "seo_geo/sitemap/registry"
+require_relative "seo_geo/page"
+# Head, JSON-LD, crawler fallback
+require_relative "seo_geo/json_ld"
+require_relative "seo_geo/meta"
+require_relative "seo_geo/crawler_fallback"
+require_relative "seo_geo/controller"
+# Robots, sitemap, llms.txt
+require_relative "seo_geo/robots"
+require_relative "seo_geo/sitemap/xml"
+require_relative "seo_geo/llms_txt"
+# Markdown, IndexNow, OG image
+require_relative "seo_geo/markdown/negotiation"
+require_relative "seo_geo/markdown/document"
+require_relative "seo_geo/markdown/path_suffix"
+require_relative "seo_geo/markdownable"
+require_relative "seo_geo/index_now"
+require_relative "seo_geo/model"
+require_relative "seo_geo/og_image/renderer"
+require_relative "seo_geo/og_image/ferrum_backend"
+# Noindex path matching, robots evaluator, audit, bot verification
+require_relative "seo_geo/path_match"
+require_relative "seo_geo/publication"
+require_relative "seo_geo/robots/evaluator"
+require_relative "seo_geo/audit"
+require_relative "seo_geo/bot_verifier"
+
+# SEO and GEO (Generative Engine Optimization: visibility in AI search and assistants) for Rails.
+# Not a geolocation library.
+module SeoGeo
+  class << self
+    # @return [Configuration]
+    def config
+      @config ||= Configuration.new
+    end
+
+    # @yieldparam config [Configuration]
+    # @return [Configuration]
+    def configure
+      yield config
+      config
+    end
+
+    # @return [Sitemap::Registry]
+    def sitemap
+      @sitemap ||= Sitemap::Registry.new
+    end
+
+    # @return [PageRegistry]
+    def pages
+      @pages ||= PageRegistry.new
+    end
+
+    # @param key [Symbol, String]
+    # @param locale [Symbol, String, nil]
+    # @return [Page]
+    def page(key, locale: nil)
+      pages.fetch(key, locale: locale)
+    end
+
+    # Replaces configuration and registries with fresh instances (tests).
+    # @return [void]
+    def reset!
+      @config = Configuration.new
+      @sitemap = Sitemap::Registry.new
+      @pages = PageRegistry.new
+    end
+  end
+end
+
+require_relative "seo_geo/engine" if defined?(Rails::Engine)

data/lib/tasks/seo_geo.rake

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+namespace :seo_geo do
+  # `rake seo_geo:audit` (requests every registered URL and reports problems).
+  # Set before :environment so the engine adds the audited hosts to config.hosts outside production
+  # (SeoGeo::Engine.allow_audit_hosts!). For `bin/rails runner`, export SEO_GEO_AUDIT=1 yourself.
+  task :audit_flag do
+    ENV["SEO_GEO_AUDIT"] = "1"
+  end
+
+  desc "Audit robots.txt, sitemap.xml, llms.txt and every sitemap page in-process " \
+       "(FORMAT=json, HOSTS=a.com,b.com, MAX_PAGES=200); exits 1 when there are errors"
+  task audit: %i[audit_flag environment] do
+    report = SeoGeo::Audit::Runner.new(
+      app: Rails.application,
+      root: Rails.root,
+      hosts: ENV["HOSTS"]&.split(",")&.map(&:strip)&.reject(&:empty?),
+      max_pages: Integer(ENV.fetch("MAX_PAGES", "200"), 10)
+    ).run
+    puts ENV["FORMAT"].to_s.casecmp?("json") ? report.to_json : report.to_s
+    exit(1) if report.errors?
+  end
+end