seo_geo 0.1.0

data/lib/seo_geo/controller.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # Included into ActionController::Base by the engine (initializer "seo_geo.helpers").
+  #
+  # @example
+  #   class PostsController < ApplicationController
+  #     def show
+  #       @post = Post.find(params[:id])
+  #       seo title: @post.title, description: @post.summary, image: @post.cover_url, type: "article"
+  #     end
+  #   end
+  #
+  #   class LandingController < ApplicationController
+  #     def show
+  #       render inertia: "Home", props: { seo: seo_geo_page(:home).props }
+  #     end
+  #   end
+  module Controller
+    SEO_KEYS = Meta::State.members.freeze
+
+    def self.included(base)
+      base.helper_method :seo_geo_state, :seo_geo_page, :seo_geo_markdown_available? if base.respond_to?(:helper_method)
+    end
+
+    # Sets head values for this request. Later calls merge into earlier ones; +json_ld:+ appends.
+    #
+    # @param title [String] page title without the site suffix (title_template adds it)
+    # @param description [String]
+    # @param image [String, Hash] absolute or root-relative OG image URL (or +{url:}+)
+    # @param noindex [Boolean] true => robots noindex
+    # @param index [Boolean] true => opt this action into indexing when +default_noindex+ is on
+    # @param canonical [String, false] path or URL to canonicalize; false omits the canonical link and hreflang
+    # @param page [Symbol, Page] page copy (title/description/FAQ JSON-LD defaults)
+    # @param type [String] og:type (default "website")
+    # @param markdown [Boolean] false hides the Markdown alternate link
+    # @param json_ld [Hash, JsonLd, Array] extra JSON-LD nodes
+    # @return [Meta::State]
+    def seo(**options)
+      unknown = options.keys - SEO_KEYS
+      raise ArgumentError, "unknown seo option(s): #{unknown.join(", ")} (known: #{SEO_KEYS.join(", ")})" unless unknown.empty?
+
+      extra_json_ld = options.delete(:json_ld)
+      options.each { |name, value| seo_geo_state[name] = value }
+      seo_geo_state.json_ld.concat(extra_json_ld.is_a?(Array) ? extra_json_ld.flatten : [extra_json_ld]) if extra_json_ld
+      seo_geo_state
+    end
+
+    # Fetches the page copy and uses it for this request's head (title, description, FAQPage).
+    # The return value feeds the crawler fallback body and Inertia props: +seo_geo_page(:home).props+.
+    # @param key [Symbol, String]
+    # @param locale [Symbol, String, nil]
+    # @return [Page]
+    def seo_geo_page(key, locale: nil)
+      page = SeoGeo.page(key, locale: locale)
+      seo_geo_state.page = page
+      page
+    end
+
+    # Whether this action answers Markdown requests; {SeoGeo::Markdownable} overrides it per action.
+    # +seo_geo_head+ advertises the Markdown alternate only when this is true.
+    # @return [Boolean]
+    def seo_geo_markdown_available?
+      false
+    end
+
+    # @return [Meta::State]
+    def seo_geo_state
+      @seo_geo_state ||= Meta::State.new
+    end
+  end
+end

data/lib/seo_geo/crawler_fallback.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # Static page body for crawlers that do not run JavaScript (GPTBot, ClaudeBot, PerplexityBot, Yeti...)
+  # on client-rendered pages (Inertia, SPA). It renders the same {Page} copy as the Inertia props and
+  # the FAQPage JSON-LD. The head snippet from +seo_geo_head+ hides it while JavaScript runs and shows it
+  # again when the app has not called +window.SeoGeo.mounted()+ within five seconds.
+  #
+  # Markup lives in +app/views/seo_geo/fallback/_body.html.erb+ and +_faq.html.erb+; an app overrides
+  # it by adding the same paths under its own +app/views+.
+  module CrawlerFallback
+    # View helpers, included into ActionView by the engine (initializer "seo_geo.helpers").
+    module Helper
+      # Extra crawler-visible HTML goes at the end of +<main>+, from the block and/or
+      # +content_for(:seo_geo_fallback_extra)+ (block first).
+      #
+      # @example
+      #   <%= seo_geo_fallback_body(:home) do %>
+      #     <p><a href="/pricing">Pricing</a></p>
+      #   <% end %>
+      # @param key [Symbol, String, Page] page key or a fetched page
+      # @param locale [Symbol, String, nil]
+      # @return [ActiveSupport::SafeBuffer] +<main data-seo-fallback>+ with heading, intro, features, FAQ, extra
+      def seo_geo_fallback_body(key, locale: nil, &block)
+        extra = safe_join([block && capture(&block), content_for?(:seo_geo_fallback_extra) ? content_for(:seo_geo_fallback_extra) : nil].compact)
+        render(partial: "seo_geo/fallback/body", locals: { page: seo_geo_fallback_page(key, locale), extra: extra })
+      end
+
+      # The FAQ section alone, for server-rendered (ERB) pages that show the FAQ themselves; it is the
+      # same text as the FAQPage JSON-LD.
+      # @param key [Symbol, String, Page]
+      # @param locale [Symbol, String, nil]
+      # @return [ActiveSupport::SafeBuffer]
+      def seo_geo_faq(key, locale: nil)
+        render(partial: "seo_geo/fallback/faq", locals: { page: seo_geo_fallback_page(key, locale) })
+      end
+
+      private
+
+      def seo_geo_fallback_page(key, locale)
+        key.is_a?(Page) ? key : SeoGeo.page(key, locale: locale)
+      end
+    end
+  end
+end

data/lib/seo_geo/crawlers.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # Canonical user-agent table, grouped by what the operator uses the fetch for.
+  #
+  # Purposes:
+  # - +:training+ — model training corpora. Blocking these never affects search inclusion.
+  # - +:search+   — search / answer-engine indexes (classic and AI search).
+  # - +:user+     — fetches triggered by a person asking an assistant about a URL.
+  #   Several operators state robots.txt may not apply to these; see {Crawler#honors_robots}.
+  #
+  # Sources (checked 2026-10-14): developers.openai.com/api/docs/bots,
+  # support.claude.com/en/articles/8896518, docs.perplexity.ai/guides/bots,
+  # developers.google.com/search/docs/crawling-indexing/google-common-crawlers,
+  # support.apple.com/en-us/119829, developers.facebook.com/docs/sharing/webmasters/web-crawlers,
+  # commoncrawl.org/ccbot, searchadvisor.naver.com/guide/seo-basic-robots.
+  module Crawlers
+    PURPOSES = %i[training search user].freeze
+
+    # One robots.txt user-agent token.
+    #
+    # @!attribute user_agent [String] token as written in a robots.txt +User-agent:+ line
+    # @!attribute owner [String] operator name
+    # @!attribute purpose [Symbol] one of {PURPOSES}
+    # @!attribute honors_robots [Boolean, Symbol] true, or +:unreliable+ / +:may_ignore+ when the
+    #   operator says (or reports show) robots.txt may not be applied
+    # @!attribute token_only [Boolean] true for control tokens that never fetch on their own
+    #   (Google-Extended, Applebot-Extended); they only appear in robots.txt
+    Crawler = Data.define(:user_agent, :owner, :purpose, :honors_robots, :token_only)
+
+    def self.entry(user_agent, owner, purpose, honors_robots: true, token_only: false)
+      Crawler.new(user_agent: user_agent, owner: owner, purpose: purpose,
+                  honors_robots: honors_robots, token_only: token_only).freeze
+    end
+    private_class_method :entry
+
+    # @return [Array<Crawler>] the full table, frozen
+    ALL = [
+      entry("GPTBot", "OpenAI", :training),
+      entry("ClaudeBot", "Anthropic", :training),
+      entry("Google-Extended", "Google", :training, token_only: true),
+      entry("Applebot-Extended", "Apple", :training, token_only: true),
+      entry("CCBot", "Common Crawl", :training),
+      entry("Meta-ExternalAgent", "Meta", :training),
+      entry("Bytespider", "ByteDance", :training, honors_robots: :unreliable),
+
+      entry("Googlebot", "Google", :search),
+      entry("Bingbot", "Microsoft", :search),
+      entry("Yeti", "Naver", :search),
+      entry("OAI-SearchBot", "OpenAI", :search),
+      entry("Claude-SearchBot", "Anthropic", :search),
+      entry("PerplexityBot", "Perplexity", :search),
+
+      entry("ChatGPT-User", "OpenAI", :user, honors_robots: :may_ignore),
+      entry("Claude-User", "Anthropic", :user),
+      entry("Perplexity-User", "Perplexity", :user, honors_robots: :may_ignore),
+      entry("Meta-ExternalFetcher", "Meta", :user, honors_robots: :may_ignore)
+    ].freeze
+
+    # @return [Hash{Symbol => Array<String>}] default user-agent tokens per purpose, frozen
+    DEFAULT_USER_AGENTS = PURPOSES.to_h do |purpose|
+      [purpose, ALL.select { |crawler| crawler.purpose == purpose }.map(&:user_agent).freeze]
+    end.freeze
+
+    # @param user_agent [String] robots.txt token (case-insensitive) or a full User-Agent header
+    # @return [Crawler, nil] the table entry whose token appears in +user_agent+
+    def self.find(user_agent)
+      return nil if user_agent.nil? || user_agent.empty?
+
+      needle = user_agent.downcase
+      exact = ALL.find { |crawler| crawler.user_agent.downcase == needle }
+      return exact if exact
+
+      # Longest token first so "Claude-SearchBot" is not matched by a shorter token.
+      ALL.sort_by { |crawler| -crawler.user_agent.length }
+         .find { |crawler| needle.include?(crawler.user_agent.downcase) }
+    end
+  end
+end

data/lib/seo_geo/engine.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module SeoGeo
+  # Mount at the root so /robots.txt, /sitemap.xml and /llms.txt resolve:
+  #   mount SeoGeo::Engine, at: "/"
+  class Engine < ::Rails::Engine
+    isolate_namespace SeoGeo
+
+    # The same object as SeoGeo.config, so `config.seo_geo.site.name = ...` in
+    # config/application.rb and `SeoGeo.configure` in an initializer are interchangeable.
+    config.seo_geo = SeoGeo.config
+
+    initializer "seo_geo.pages" do |app|
+      default = app.root.join("config/seo_geo/pages").to_s
+      SeoGeo.pages.paths |= [default]
+    end
+
+    # initializer "seo_geo.helpers" (ActionView/ActionController on_load hooks).
+    initializer "seo_geo.helpers" do
+      ActiveSupport.on_load(:action_view) do
+        include SeoGeo::Meta::Helper
+        include SeoGeo::CrawlerFallback::Helper
+      end
+      ActiveSupport.on_load(:action_controller_base) { include SeoGeo::Controller }
+    end
+
+    # initializer "seo_geo.mime" (Mime::Type.register "text/markdown", :md).
+    initializer "seo_geo.mime" do |app|
+      Mime::Type.register "text/markdown", :md unless Mime::Type.lookup_by_extension(:md)
+      # /<path>.md and /index.md for every route; appended, so ActionDispatch::Static (public/*.md) wins.
+      app.middleware.use SeoGeo::Markdown::PathSuffix
+    end
+
+    # IndexNow job (only when ActiveJob is loaded) and the seo_geo_indexnow model macro.
+    initializer "seo_geo.index_now" do
+      Rails.autoloaders.main.ignore(root.join("app/jobs")) unless defined?(ActiveJob::Base)
+      ActiveSupport.on_load(:active_record) { extend SeoGeo::Model }
+    end
+
+    config.after_initialize do
+      SeoGeo::Engine.check_configuration!
+    end
+
+    # +rake seo_geo:audit+ requests the production host names in-process. Outside production Rails
+    # HostAuthorization (config.hosts, e.g. [".localhost"]) would answer 403 for them, and config.hosts is copied
+    # when the middleware stack is built — so with SEO_GEO_AUDIT=1 (set by the rake task; set it yourself for
+    # +bin/rails runner+) the configured hosts are appended here, after the app's initializers and before the stack
+    # is built. Never in production, never without the flag, never when config.hosts is empty (allows all).
+    AUDIT_FLAG = "SEO_GEO_AUDIT"
+
+    initializer "seo_geo.audit_hosts", after: :load_config_initializers, before: :build_middleware_stack do |app|
+      SeoGeo::Engine.allow_audit_hosts!(app.config)
+    end
+
+    # @param app_config [Rails::Application::Configuration]
+    # @param env [Hash] process environment (the flag)
+    # @param production [Boolean]
+    # @return [Array<String>] hosts appended to +config.hosts+ (empty when nothing was changed)
+    def self.allow_audit_hosts!(app_config, env: ENV, production: Rails.env.production?, logger: Rails.logger)
+      return [] if production || env[AUDIT_FLAG] != "1"
+      return [] unless app_config.respond_to?(:hosts) && !app_config.hosts.empty?
+
+      added = SeoGeo.config.known_hosts - app_config.hosts.map(&:to_s)
+      return [] if added.empty?
+
+      app_config.hosts.concat(added)
+      message = "seo_geo audit: temporarily allowing #{added.join(", ")} in config.hosts"
+      logger&.info(message)
+      Kernel.warn(message)
+      added
+    end
+
+    MISSING_ORIGIN = "seo_geo: canonical_origin is not set, so every host is fail-closed (Disallow: /, noindex). " \
+                     "Run bin/rails g seo_geo:install https://www.example.com or set it in config/initializers/seo_geo.rb."
+
+    PRODUCTION_TOOLING = "seo_geo: invalid configuration ignored while a generator or rake task runs in production " \
+                         "(a production server boot raises):"
+
+    # Boot-time validation that cannot lock a fresh app out of its own installer or build:
+    # - production server boot: {Configuration#validate!} raises on any problem;
+    # - production generator/rake task (+assets:precompile+, +db:migrate+ in a Docker build where the origin comes
+    #   from runtime ENV): warns and continues;
+    # - generators and rake tasks elsewhere: skipped (+rails g seo_geo:install+ must boot before the initializer exists);
+    # - otherwise a missing +canonical_origin+ only logs a warning; other problems still raise.
+    # @return [void]
+    def self.check_configuration!(config: SeoGeo.config, production: Rails.env.production?, tooling: tooling?,
+                                  logger: Rails.logger)
+      if production
+        return config.validate! unless tooling
+
+        problems = config.errors
+        unless problems.empty?
+          message = "#{PRODUCTION_TOOLING}\n- #{problems.join("\n- ")}"
+          logger&.warn(message)
+          Kernel.warn(message)
+        end
+        return
+      end
+      return if tooling
+
+      if config.canonical_origin.nil?
+        logger&.warn(MISSING_ORIGIN)
+        Kernel.warn(MISSING_ORIGIN)
+        problems = config.errors.reject { |message| message.start_with?("canonical_origin is required") }
+        raise ConfigurationError, "invalid seo_geo configuration:\n- #{problems.join("\n- ")}" unless problems.empty?
+        return
+      end
+      config.validate!
+    end
+
+    # @return [Boolean] true while +rails generate/destroy+ boots the app (their command classes are loaded only
+    # for those commands, before boot) or a rake task runs (+rails db:migrate+, +rake ...+). Merely having
+    # +rails/generators+ loaded does not count.
+    def self.tooling?
+      return true if defined?(::Rails::Command::GenerateCommand) || defined?(::Rails::Command::DestroyCommand)
+      return false unless defined?(::Rake) && ::Rake.respond_to?(:application)
+
+      !::Rake.application.top_level_tasks.empty?
+    end
+
+    # lib/tasks/*.rake is loaded by Rails::Engine itself; loading it again here ran every task twice.
+  end
+end

data/lib/seo_geo/index_now.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "json"
+require "logger"
+require "net/http"
+require "openssl"
+require "uri"
+
+module SeoGeo
+  # IndexNow (https://www.indexnow.org/documentation): tells Bing, Naver, Yandex, Seznam and Yep that
+  # URLs changed. One POST to the shared endpoint reaches every participant. Google does not take part.
+  #
+  # @example
+  #   SeoGeo.configure do |c|
+  #     c.indexnow.enabled = true
+  #     c.indexnow.key = ENV["INDEXNOW_KEY"]   # also served at /<key>.txt
+  #   end
+  #   SeoGeo::IndexNow.submit(["/articles/1", "https://www.example.com/articles/2"])
+  #
+  # Outside indexable environments (or with +indexnow.dry_run = true+) the payload is logged, not sent.
+  module IndexNow
+    # Base class for delivery failures.
+    class Error < StandardError
+      # @return [Integer, nil] HTTP status
+      attr_reader :status
+
+      def initialize(message, status: nil)
+        super(message)
+        @status = status
+      end
+    end
+
+    # 429, 5xx and network errors: the job retries with backoff.
+    class RetryableError < Error; end
+
+    # 400 bad request, 403 key not valid, 422 URLs do not belong to the host: retrying cannot help.
+    class RejectedError < Error; end
+
+    # Names other routes serve as +/<name>.txt+; never accepted as a key.
+    RESERVED_KEYS = %w[robots llms llms-full humans security ads app-ads].freeze
+
+    # Route constraint for the key file: matches only while a valid, non-reserved key is configured
+    # and equals the requested one, so /robots.txt, /llms-full.txt and other +.txt+ paths are never caught.
+    module KeyConstraint
+      # @param request [ActionDispatch::Request]
+      # @return [Boolean]
+      def self.matches?(request)
+        key = SeoGeo.config.indexnow.key.to_s
+        IndexNowOptions::KEY_FORMAT.match?(key) && !RESERVED_KEYS.include?(key.downcase) &&
+          request.path_parameters[:indexnow_key] == key
+      end
+    end
+
+    # Outcome of one POST (or one dry run).
+    # @!attribute host [String]
+    # @!attribute urls [Array<String>]
+    # @!attribute status [Integer, nil] HTTP status; nil for a dry run
+    # @!attribute dry_run [Boolean]
+    Result = Data.define(:host, :urls, :status, :dry_run)
+
+    REJECTED = {
+      400 => "bad request (invalid payload format)",
+      403 => "key not valid (key file missing or its content does not match)",
+      422 => "URLs do not belong to the host, or the key does not match the protocol schema"
+    }.freeze
+
+    class << self
+      # @return [Logger]
+      attr_writer :logger
+
+      # @return [Logger] Rails.logger when available
+      def logger
+        @logger || (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) || Logger.new($stderr)
+      end
+
+      # Normalizes, de-duplicates and groups URLs by host, then enqueues one {IndexNowJob} per batch
+      # (or delivers inline when ActiveJob is not loaded).
+      # @param urls [String, Array<String>] absolute URLs or paths; paths and alias hosts fold onto
+      #   the canonical origin via +config.canonical_url+
+      # @param config [Configuration]
+      # @return [Array<Array(String, Array<String>)>] the +[host, urls]+ batches handed off; empty when disabled
+      def submit(urls, config: SeoGeo.config)
+        return [] unless config.indexnow.enabled
+
+        batches(urls, config: config).each do |host, list|
+          if job_class
+            job_class.perform_later(host, list)
+          else
+            deliver(host, list, config: config)
+          end
+        end
+      end
+
+      # @param urls [String, Array<String>]
+      # @param config [Configuration]
+      # @return [Array<Array(String, Array<String>)>] +[host, urls]+ pairs of at most +batch_size+ URLs
+      def batches(urls, config: SeoGeo.config)
+        grouped = normalize(urls, config).group_by { |url| URI.parse(url).host }
+        grouped.flat_map do |host, list|
+          if config.indexable_environment? && !config.host_policy(host).indexable
+            logger.info("[seo_geo] IndexNow skipped #{list.size} URL(s) on non-indexable host #{host}")
+            next []
+          end
+          list.each_slice(config.indexnow.batch_size).map { |slice| [host, slice] }
+        end
+      end
+
+      # @param host [String]
+      # @param urls [Array<String>]
+      # @param config [Configuration]
+      # @return [Hash] the JSON body defined by the protocol
+      def payload(host, urls, config: SeoGeo.config)
+        key = config.indexnow.key
+        { host: host, key: key, keyLocation: "#{config.origin_for(host)}/#{key}.txt", urlList: urls }
+      end
+
+      # @param config [Configuration]
+      # @return [Boolean] +indexnow.dry_run+, or true outside indexable environments when it is nil
+      def dry_run?(config: SeoGeo.config)
+        value = config.indexnow.dry_run
+        value.nil? ? !config.indexable_environment? : value
+      end
+
+      # POSTs one batch. Called by {IndexNowJob}; usable directly for synchronous delivery.
+      # @param host [String]
+      # @param urls [Array<String>]
+      # @param config [Configuration]
+      # @raise [RetryableError] on 429, 5xx or a network error
+      # @raise [RejectedError] on 400, 403, 422 or another unexpected status
+      # @return [Result]
+      def deliver(host, urls, config: SeoGeo.config)
+        body = JSON.generate(payload(host, urls, config: config))
+        if dry_run?(config: config)
+          logger.info("[seo_geo] IndexNow dry-run POST #{config.indexnow.endpoint} #{body}")
+          return Result.new(host: host, urls: urls, status: nil, dry_run: true)
+        end
+
+        status = post(config.indexnow.endpoint, body)
+        handle(status, host, urls)
+      end
+
+      private
+
+      def job_class
+        defined?(ActiveJob::Base) && defined?(SeoGeo::IndexNowJob) ? SeoGeo::IndexNowJob : nil
+      end
+
+      def normalize(urls, config)
+        Array(urls).flatten.compact.map(&:to_s).reject(&:empty?).filter_map do |url|
+          folded = config.canonical_url(url)
+          folded if folded.start_with?("http://", "https://")
+        rescue URI::InvalidURIError
+          logger.warn("[seo_geo] IndexNow ignored invalid URL #{url.inspect}")
+          nil
+        end.uniq
+      end
+
+      def post(endpoint, body)
+        uri = URI.parse(endpoint)
+        request = Net::HTTP::Post.new(uri)
+        request["Content-Type"] = "application/json; charset=utf-8"
+        request.body = body
+        response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                                       open_timeout: 5, read_timeout: 15) do |http|
+          http.request(request)
+        end
+        response.code.to_i
+      rescue IOError, SystemCallError, Timeout::Error, SocketError, OpenSSL::SSL::SSLError => e
+        raise RetryableError, "IndexNow request failed: #{e.class}: #{e.message}"
+      end
+
+      def handle(status, host, urls)
+        case status
+        when 200, 202
+          logger.info("[seo_geo] IndexNow #{status} for #{urls.size} URL(s) on #{host}")
+          Result.new(host: host, urls: urls, status: status, dry_run: false)
+        when 429, 500..599
+          raise RetryableError.new("IndexNow #{status} for #{host} (#{status == 429 ? "too many requests" : "server error"}); will retry", status: status)
+        else
+          reason = REJECTED.fetch(status, "unexpected response")
+          logger.error("[seo_geo] IndexNow #{status} for #{host}: #{reason}; not retrying")
+          raise RejectedError.new("IndexNow #{status} for #{host}: #{reason}", status: status)
+        end
+      end
+    end
+  end
+end