seo_geo 0.1.0

data/lib/generators/seo_geo/install/install_generator.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module SeoGeo
+  module Generators
+    # +bin/rails g seo_geo:install [CANONICAL_ORIGIN]+: initializer, sample page copy, engine
+    # mount, layout instructions, and a warning for static files that would shadow the engine routes.
+    class InstallGenerator < ::Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      SHADOWED_FILES = %w[robots.txt sitemap.xml llms.txt llms-full.txt].freeze
+      MOUNT = 'mount SeoGeo::Engine, at: "/"'
+      # A catch-all route (+get "*path"+, +match "/*slug"+, ...).
+      CATCH_ALL = /\A[ \t]*(?:get|match|any|post)\b[ \t(]*["']\/?\*/
+      DRAW = /\.routes\.draw\s+do\b/
+      # Lines that open a block closed by +end+: a trailing +do+ (optionally +do |args|+) or a leading keyword.
+      OPENS_BLOCK = /(?:\bdo(?:\s*\|[^|]*\|)?\s*(?:#.*)?\z)|\A\s*(?:if|unless|case|begin|while|until|def|class|module)\b/
+      CLOSES_BLOCK = /\A\s*end\b/
+
+      # @param lines [Array<String>] config/routes.rb
+      # @return [Integer, nil] line index for the mount: directly inside +routes.draw+ (never inside a scope or
+      #   namespace), before its first top-level catch-all, else before the +end+ closing the draw block
+      def self.mount_index(lines)
+        start = lines.index { |line| line.match?(DRAW) }
+        return nil if start.nil?
+
+        depth = 1
+        lines.each_with_index.drop(start + 1).each do |line, index|
+          return index if depth == 1 && line.match?(CATCH_ALL)
+
+          if line.match?(CLOSES_BLOCK)
+            depth -= 1
+            return index if depth.zero?
+          elsif line.match?(OPENS_BLOCK)
+            depth += 1
+          end
+        end
+        nil
+      end
+
+      argument :canonical_origin, type: :string, default: "https://www.example.com",
+                                  banner: "https://www.example.com"
+
+      class_option :site_name, type: :string, default: nil, desc: "site.name (defaults to the application name)"
+
+      def validate_origin
+        return if canonical_origin.match?(%r{\Ahttps?://[^/\s]+/?\z})
+
+        raise Thor::Error, "CANONICAL_ORIGIN must look like https://www.example.com (got #{canonical_origin.inspect})"
+      end
+
+      def create_initializer
+        template "initializer.rb.tt", "config/initializers/seo_geo.rb"
+      end
+
+      def create_home_page
+        template "home.yml.tt", "config/seo_geo/pages/home.yml"
+      end
+
+      # The mount goes last (before a top-level catch-all +get "*path"+ if there is one), not at the top where Rails'
+      # +route+ puts it: routes match in order, so the app's own routes (e.g. +/og/:username.png+) stay first. One
+      # insertion point, directly inside +routes.draw+; an existing mount is left alone.
+      def mount_engine
+        routes = File.join(destination_root, "config/routes.rb")
+        return route(MOUNT) unless File.exist?(routes)
+
+        content = File.read(routes)
+        if content.include?("SeoGeo::Engine")
+          say_status :identical, "config/routes.rb (SeoGeo::Engine already mounted)", :blue
+          return
+        end
+
+        lines = content.lines
+        index = self.class.mount_index(lines.map(&:chomp))
+        return route(MOUNT) if index.nil?
+
+        catch_all = lines[index].match?(CATCH_ALL)
+        indent = lines[index][/\A[ \t]*/]
+        indent = "#{indent}  " unless catch_all
+        lines.insert(index, "#{indent}#{MOUNT}\n")
+        say_status :insert, "config/routes.rb", :green
+        File.write(routes, lines.join) unless options[:pretend]
+        say_status :note, "mounted SeoGeo::Engine after your routes#{catch_all ? " (before the catch-all)" : ""}: " \
+                          "app routes win where paths overlap, and /robots.txt, /sitemap.xml, /llms.txt still reach " \
+                          "the engine. Keep any catch-all route below the mount.", :blue
+      end
+
+      def warn_about_static_files
+        SHADOWED_FILES.each do |name|
+          next unless File.exist?(File.join(destination_root, "public", name))
+
+          say_status :warning, "public/#{name} is served before the seo_geo route — delete it " \
+                               "(rake seo_geo:audit reports it as an error)", :yellow
+        end
+      end
+
+      def show_layout_instructions
+        say <<~TEXT
+
+          seo_geo is installed. In app/views/layouts/application.html.erb:
+
+            <html lang="<%= seo_geo_html_lang %>">
+              <head>
+                <%= seo_geo_head %>          <!-- replaces your <title>, meta description, canonical, JSON-LD -->
+              </head>
+
+          Remove any other <title> tag (seo_geo_head emits exactly one).
+
+          Client-rendered pages (Inertia/React/Vue): render the copy for crawlers that do not run JavaScript
+          inside <body>, and call window.SeoGeo.mounted() once the app has mounted:
+
+            <%= seo_geo_fallback_body(:home) %>
+
+          In controllers: seo title: "...", description: "..." or seo_geo_page(:home).
+          Then run: bin/rails seo_geo:audit
+        TEXT
+      end
+
+      private
+
+      def origin
+        canonical_origin.chomp("/")
+      end
+
+      def site_name
+        options[:site_name] || default_site_name
+      end
+
+      def default_site_name
+        name = defined?(::Rails.application) && ::Rails.application ? ::Rails.application.class.module_parent_name : nil
+        name ? name.underscore.humanize : "My site"
+      rescue StandardError
+        "My site"
+      end
+    end
+  end
+end

data/lib/generators/seo_geo/install/templates/home.yml.tt

@@ -0,0 +1,14 @@
+# Page copy for SeoGeo.page(:home): title/description feed the head, faq feeds both the visible FAQ
+# (seo_geo_faq / seo_geo_fallback_body) and the FAQPage JSON-LD, so they always match.
+# Keyed by locale; a flat file (no locale keys) also works.
+en:
+  title: <%= site_name.to_json %>
+  description: Replace this with a 70–160 character summary of the home page for search results.
+  heading: <%= site_name.to_json %>
+  intro: One or two sentences that say what the site does and for whom.
+  features:
+    - title: First feature
+      description: What it does for the reader.
+  faq:
+    - question: <%= "What is #{site_name}?".to_json %>
+      answer: A short, factual answer that is also shown on the page.

data/lib/generators/seo_geo/install/templates/initializer.rb.tt

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# seo_geo: SEO and GEO (AI search visibility). Every option is documented in docs/DESIGN.md of the gem.
+SeoGeo.configure do |seo|
+  seo.site.name = <%= site_name.inspect %>
+  seo.site.description = "One sentence that describes the site."
+  # seo.site.logo_url = "<%= origin %>/logo.png"
+  # seo.site.default_locale = :en   # <html lang>, inLanguage and og:locale (:ko => ko_KR) use this
+  # seo.site.same_as = ["https://github.com/your-org"]
+
+  # One canonical origin; alias hosts are folded onto it, undeclared hosts get Disallow: /.
+  seo.canonical_origin = <%= origin.inspect %>
+  # seo.canonical_aliases = ["example.com"]  # aliases inherit the canonical host's disallow/allow
+  # seo.host "www.example.com", disallow: ["/admin", "/oauth/"]
+  # seo.share_prefixes = ["/s/"]           # open for link-preview bots
+  # seo.noindex_paths = ["/waitlist", %r{\A/drafts/}]  # robots meta, sitemap.xml and llms.txt agree
+
+  # Crawlers by purpose (:allow or :deny): training, search, user-triggered fetches.
+  # seo.crawlers.training = :deny
+  # seo.crawlers.override "Bytespider", :deny
+  # seo.crawlers.override "Mediapartners-Google", :full_access  # Allow: / — ignores host disallows (opt-in)
+
+  # Search-console verification tokens.
+  # seo.verification.google = "..."
+  # seo.verification.bing = "..."
+  # seo.verification.naver = "..."
+
+  # seo.locales = %i[en ko]   # only then does I18n.locale pick the page language (else default_locale)
+  # seo.title_template = "%{title} | %{site}"
+  # seo.default_noindex = false
+
+  # GET /og/:key.png is routed only once a backend (or template) is set, and only for known keys
+  # (page keys, og_image.keys, og_image.key_resolver); other /og/... URLs reach your own routes.
+  # seo.og_image.backend = ->(key, page, params) { MyOgRenderer.png(key, page) }
+  # seo.og_image.fallback_url = "<%= origin %>/og-default.png"
+  # seo.og_image.fallback_width = 1200   # og:image:width/height for the fallback only when both are set
+  # seo.og_image.fallback_height = 630
+end
+
+# Sitemap sources. Never pass Time.now as lastmod; use real date columns.
+# Collections: pass a lambda so no model is loaded at boot and dev reloads never keep a stale class;
+# it is called per sitemap request. Records are read with find_each (select must include :id, order is
+# ignored); add batch: false to keep the scope's order.
+SeoGeo.sitemap.draw do
+  add "/", priority: 1.0
+  # add -> { Post.published }, path: ->(post) { "/posts/#{post.slug}" }, lastmod: :updated_at
+  # Under default_noindex: add ..., index: true, match: %r{\A/posts/[^/]+\z}   # head of each post indexable
+  # exclude %r{\A/s/}
+end
+
+# llms.txt: list the pages agents should read first (page copy comes from config/seo_geo/pages/*.yml).
+SeoGeo.config.llms_txt.page :home, path: "/"

data/lib/seo_geo/audit/checks.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "uri"
+require "nokogiri"
+
+module SeoGeo
+  module Audit
+    # Pure checks over response bodies. Shared by {Runner}, {SeoGeo::Testing::Assertions} and the
+    # RSpec matchers, so a test and the audit can never disagree. Each returns problem messages
+    # (empty when fine).
+    module Checks
+      TITLE_MAX = 60
+      DESCRIPTION_RANGE = (70..160)
+      SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
+
+      module_function
+
+      # @param doc [HtmlDocument]
+      # @return [Array<String>] problems with +<title>+ count (errors)
+      def title_count(doc)
+        count = doc.titles.size
+        return [] if count == 1
+
+        [count.zero? ? "no <title>" : "#{count} <title> elements (exactly one expected)"]
+      end
+
+      # @param doc [HtmlDocument]
+      # @return [Array<String>] FAQPage questions or answers missing from the visible text
+      def faq_visible(doc)
+        text = doc.visible_text
+        doc.faq_items.flat_map do |item|
+          problems = []
+          problems << "FAQ question not in visible text: #{item[:question].inspect}" unless text.include?(item[:question])
+          problems << "FAQ answer not in visible text: #{item[:answer].inspect}" unless text.include?(item[:answer])
+          problems
+        end
+      end
+
+      # @param doc [HtmlDocument]
+      # @return [Array<String>] unparsable blocks and nodes missing properties {JsonLd::REQUIRED} lists
+      def json_ld(doc)
+        nodes, problems = doc.json_ld
+        nodes.each do |node|
+          doc.types(node).each do |type|
+            missing = JsonLd::REQUIRED.fetch(type, []).select { |key| blank?(node[key]) }
+            problems << [type, node["@id"], "is missing #{missing.join(", ")}"].compact.join(" ") unless missing.empty?
+          end
+        end
+        problems
+      end
+
+      # @param xml [String] sitemap.xml body
+      # @param origin [String, nil] every +<loc>+ must start with it (e.g. "https://www.example.com/")
+      # @return [Array(Array<String>, Array<String>, Symbol)] problems, +<loc>+ values, :urlset/:sitemapindex
+      def sitemap(xml, origin: nil)
+        doc = Nokogiri::XML(xml.to_s) { |options| options.strict.nonet }
+        root = doc.root
+        kind = root&.name&.to_sym
+        unless %i[urlset sitemapindex].include?(kind) && root.namespace&.href == SITEMAP_NS
+          return [["root element must be <urlset> or <sitemapindex> in #{SITEMAP_NS}"], [], kind]
+        end
+
+        locs = doc.xpath("//s:loc", "s" => SITEMAP_NS).map { |node| node.text.strip }
+        problems = []
+        problems << "sitemap lists no URL" if locs.empty?
+        problems << "#{locs.size} URLs in one file (limit 50,000)" if locs.size > 50_000
+        locs.each do |loc|
+          problems << "<loc> #{loc} is not absolute" unless loc.match?(%r{\Ahttps?://}i)
+          problems << "<loc> #{loc} is not under #{origin}" if origin && !same_origin?(loc, origin)
+        end
+        [problems, locs, kind]
+      rescue Nokogiri::XML::SyntaxError => e
+        [["sitemap is not well-formed XML: #{e.message.strip}"], [], nil]
+      end
+
+      # @param canonical [String, nil] canonical href
+      # @param expected_origin [String] e.g. "https://www.example.com"
+      # @return [Array<String>]
+      def canonical_origin(canonical, expected_origin)
+        return ["no <link rel=\"canonical\">"] if canonical.nil? || canonical.strip.empty?
+
+        uri = URI.parse(canonical)
+        actual = uri.host && "#{uri.scheme}://#{uri.host}#{":#{uri.port}" unless [80, 443].include?(uri.port)}"
+        actual&.casecmp?(expected_origin.chomp("/")) ? [] : ["canonical #{canonical} is not on #{expected_origin}"]
+      rescue URI::InvalidURIError
+        ["canonical #{canonical.inspect} is not a valid URL"]
+      end
+
+      # @param url [String]
+      # @param origin [String]
+      # @return [Boolean] same scheme, host and effective port
+      def same_origin?(url, origin)
+        a = URI.parse(url)
+        b = URI.parse(origin)
+        a.host && b.host && a.scheme&.downcase == b.scheme&.downcase && a.host.casecmp?(b.host) && a.port == b.port
+      rescue URI::InvalidURIError
+        false
+      end
+
+      # @param value [Object]
+      # @return [Boolean]
+      def blank?(value)
+        value.nil? || (value.respond_to?(:empty?) && value.empty?) || (value.is_a?(String) && value.strip.empty?)
+      end
+    end
+  end
+end

data/lib/seo_geo/audit/html_document.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+require "nokogiri"
+
+module SeoGeo
+  module Audit
+    # Read-only view of one HTML response: the values crawlers take from it.
+    class HtmlDocument
+      NON_VISIBLE = "script, style, template, head"
+
+      # @param html [String]
+      def initialize(html)
+        @doc = Nokogiri::HTML(html.to_s)
+      end
+
+      # @return [Array<String>] text of every +<title>+ in the head (SVG titles ignored)
+      def titles
+        @doc.xpath("//head/title").map { |node| squish(node.text) }
+      end
+
+      # @return [String, nil] the first title
+      def title
+        titles.first
+      end
+
+      # @return [String, nil]
+      def description
+        meta_content("description")
+      end
+
+      # @return [String, nil] href of +<link rel="canonical">+
+      def canonical
+        @doc.css("link[rel~=canonical]").first&.[]("href")
+      end
+
+      # @return [String, nil] first robots meta content (lower case); {#noindex?} reads all of them
+      def robots
+        meta_content("robots")&.downcase
+      end
+
+      # @return [RobotsDirectives] every robots and crawler-specific meta ({RobotsDirectives.from_html}, the
+      #   parser the Markdown response uses)
+      def robots_directives
+        @robots_directives ||= RobotsDirectives.from_html(@doc)
+      end
+
+      # @param header [String, Array<String>, nil] the response's +X-Robots-Tag+
+      # @return [Boolean] true when any meta or the header tells any crawler noindex (+none+ included)
+      def noindex?(header = nil)
+        robots_directives.merge(RobotsDirectives.parse_header(header)).noindex_for_any?
+      end
+
+      # @return [String, nil] +<html lang>+
+      def html_lang
+        value = @doc.at_xpath("/html")&.[]("lang")
+        value.nil? || value.strip.empty? ? nil : value
+      end
+
+      # @return [String, nil] href of +<link rel="alternate" type="text/markdown">+
+      def markdown_alternate
+        @doc.css("link[rel~=alternate]").find { |node| node["type"].to_s.casecmp?("text/markdown") }&.[]("href")
+      end
+
+      # @param name [String] meta name, e.g. "google-site-verification"
+      # @return [String, nil]
+      def meta_content(name)
+        node = @doc.css("meta[name]").find { |item| item["name"].casecmp?(name) }
+        node && node["content"]
+      end
+
+      # @return [Array<String>] raw bodies of +<script type="application/ld+json">+
+      def json_ld_sources
+        @doc.css("script").select { |node| node["type"].to_s.strip.casecmp?("application/ld+json") }.map(&:text)
+      end
+
+      # @return [Array(Array<Hash>, Array<String>)] parsed nodes (@graph flattened) and parse error messages
+      def json_ld
+        nodes = []
+        errors = []
+        json_ld_sources.each_with_index do |source, index|
+          nodes.concat(flatten(JSON.parse(source)))
+        rescue JSON::ParserError => e
+          errors << "JSON-LD block #{index + 1} does not parse: #{e.message.lines.first.to_s.strip[0, 120]}"
+        end
+        [nodes, errors]
+      end
+
+      # @return [Array<Hash{Symbol => String}>] +{question:, answer:}+ from every FAQPage node
+      def faq_items
+        json_ld.first.select { |node| types(node).include?("FAQPage") }.flat_map do |node|
+          Array(node["mainEntity"]).filter_map do |question|
+            next unless question.is_a?(Hash)
+
+            answer = question["acceptedAnswer"]
+            answer = answer.first if answer.is_a?(Array)
+            { question: squish(question["name"].to_s), answer: squish(answer.is_a?(Hash) ? answer["text"].to_s : "") }
+          end
+        end
+      end
+
+      # @return [String] text a crawler that does not run JavaScript reads (scripts, styles, head removed)
+      def visible_text
+        @visible_text ||= begin
+          copy = @doc.dup
+          copy.css(NON_VISIBLE).each(&:remove)
+          squish(copy.text)
+        end
+      end
+
+      # @param node [Hash]
+      # @return [Array<String>]
+      def types(node)
+        Array(node["@type"]).map(&:to_s)
+      end
+
+      private
+
+      def flatten(value)
+        case value
+        when Array then value.flat_map { |item| flatten(item) }
+        when Hash then value.key?("@graph") ? flatten(value["@graph"]) : [value]
+        else []
+        end
+      end
+
+      def squish(text)
+        text.to_s.gsub(/[[:space:]]+/, " ").strip
+      end
+    end
+  end
+end

data/lib/seo_geo/audit/report.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SeoGeo
+  module Audit
+    SEVERITIES = %i[error warning].freeze
+
+    # One problem.
+    #
+    # @!attribute severity [Symbol] :error (exit non-zero) or :warning
+    # @!attribute check [String] check id, e.g. "title.count"
+    # @!attribute host [String, nil]
+    # @!attribute url [String, nil] URL or file:line
+    # @!attribute message [String]
+    Finding = Data.define(:severity, :check, :host, :url, :message) do
+      # @return [Hash{String => String}]
+      def to_h
+        { "severity" => severity.to_s, "check" => check, "host" => host, "url" => url, "message" => message }
+      end
+    end
+
+    # Collected findings plus counters. +to_s+ for people, +to_json+ for CI.
+    class Report
+      # @return [Array<Finding>]
+      attr_reader :findings
+      # @return [Hash{String => Integer}] e.g. "pages" => 12
+      attr_reader :stats
+
+      def initialize
+        @findings = []
+        @stats = Hash.new(0)
+      end
+
+      # @param severity [Symbol] :error or :warning
+      # @param check [String]
+      # @param message [String]
+      # @param host [String, nil]
+      # @param url [String, nil]
+      # @return [Finding]
+      def add(severity, check, message, host: nil, url: nil)
+        raise ArgumentError, "unknown severity #{severity.inspect}" unless SEVERITIES.include?(severity)
+
+        Finding.new(severity: severity, check: check, host: host, url: url, message: message).tap { |item| @findings << item }
+      end
+
+      # @return [Array<Finding>]
+      def errors
+        findings.select { |item| item.severity == :error }
+      end
+
+      # @return [Array<Finding>]
+      def warnings
+        findings.select { |item| item.severity == :warning }
+      end
+
+      # @return [Boolean]
+      def errors?
+        !errors.empty?
+      end
+
+      # @return [Hash]
+      def to_h
+        { "errors" => errors.size, "warnings" => warnings.size, "stats" => stats.dup,
+          "findings" => findings.map(&:to_h) }
+      end
+
+      # @return [String]
+      def to_json(*args)
+        JSON.pretty_generate(to_h, *args)
+      end
+
+      # @return [String] one line per finding, then a summary line
+      def to_s
+        lines = findings.sort_by { |item| [SEVERITIES.index(item.severity), item.host.to_s, item.check] }.map do |item|
+          where = [item.host, item.url].compact.join(" ")
+          "#{item.severity.to_s.upcase.ljust(7)} #{item.check.ljust(22)} #{where}#{where.empty? ? "" : " — "}#{item.message}"
+        end
+        counted = stats.map { |name, count| "#{count} #{name}" }.join(", ")
+        lines << "seo_geo audit: #{errors.size} error(s), #{warnings.size} warning(s)#{counted.empty? ? "" : " (#{counted})"}"
+        "#{lines.join("\n")}\n"
+      end
+    end
+  end
+end