indexmap 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3d316abdab0a6a55f613b44a0481bdf98d7d041d3d5b60b8523524450494333
-  data.tar.gz: a18dd5ad2e6db70ecef78719d8bae1a6d249fd393fe7dc363a8862fedd2faede
+  metadata.gz: 52c07631052b8a72f3745d578239e22f97d047a517d7e1f9bdd8172a98cef453
+  data.tar.gz: 6ee1835d41386d143dbea600f81e48eff7d147ddd29ad806797195ff8eb3b0e1
 SHA512:
-  metadata.gz: 8dfd199b1b978991b703870f4eb08091b8ce4689156840b7f30a985f951d55d163dce89ba71d7d8d4645931b2cb6980a4134f9b593bd8bde3b97cf272fdfbe4c
-  data.tar.gz: 875b0e45a6d3ad629f03456f62b8311f6379f3d8cde8a308ec66e6bf5233cb122a34d39531d7a245b5b7e3a3e2bc051ee7fd16d5f2981fb5e5eb6b600ef01fa8
+  metadata.gz: 6b66319af63650f80ac5139cc0502cac097ec93d9d0613f0d856db1b695aa7c6b28a7f0d525eb0513bbd2b5989becae575baf00933fcfcf2b4cceec9388a054b
+  data.tar.gz: 01130bad021a01134530cbcd1e389e1d25c69637d40fa8f31b4023fbf4015b4caaf79c55895936e76cb54e493495aa35d75a7b86d3a5b77f452b7ba945d37059

data/CHANGELOG.md

@@ -5,9 +5,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.2.1] - 2026-04-21
+## [0.3.0] - 2026-04-22
 
-### <!-- 1 -->🐛 Bug Fixes
-- publish built gem in release workflow
+### <!-- 0 -->🚀 Features
+- expand indexmap with sitemap parsing, validation, and search engine pinging
 
 

data/README.md

@@ -104,6 +104,47 @@ end
 
 In `:single_file` mode, `indexmap` writes a `urlset` directly to `sitemap.xml`. In the default `:index` mode, it writes a sitemap index plus child sitemap files from `sections`.
 
+## Validation and Parsing
+
+`indexmap` also includes small utilities for working with generated sitemap files:
+
+```ruby
+parser = Indexmap::Parser.new(path: Rails.public_path.join("sitemap.xml"))
+parser.paths
+# => ["/", "/about", "/articles/example"]
+
+Indexmap::Validator.new.validate!
+```
+
+The built-in validator checks for:
+
+- missing sitemap files
+- duplicate sitemap URLs
+- parameterized URLs in sitemap entries
+
+## Search Engine Ping
+
+The gem can ping Google Search Console and IndexNow once your app config provides the required credentials.
+
+```ruby
+Indexmap.configure do |config|
+  config.google.credentials = -> { ENV["GOOGLE_SITEMAP"] }
+  config.index_now.key = -> { ENV["INDEXNOW_KEY"] }
+end
+```
+
+When `config.index_now.key` is set, `sitemap:create` also writes the matching `public/<key>.txt` verification file automatically.
+
+Available rake tasks:
+
+```bash
+bin/rails sitemap:validate
+bin/rails sitemap:google:ping
+bin/rails sitemap:index_now:ping
+bin/rails sitemap:ping
+bin/rails sitemap:index_now:write_key
+```
+
 ## Development
 
 Run tests:

data/lib/indexmap/configuration.rb

@@ -24,10 +24,18 @@ module Indexmap
       value.nil? ? :index : value.to_sym
     end
 
+    def google
+      @google ||= GoogleConfiguration.new
+    end
+
     def index_filename
       resolve(@index_filename)
     end
 
+    def index_now
+      @index_now ||= IndexNowConfiguration.new
+    end
+
     def public_path
       value = resolve(@public_path)
       return Pathname("public") if value.nil?

data/lib/indexmap/google_configuration.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Indexmap
+  class GoogleConfiguration
+    attr_writer :credentials, :property
+
+    def credentials
+      resolve(@credentials)
+    end
+
+    def property
+      resolve(@property)
+    end
+
+    private
+
+    def resolve(value)
+      value.respond_to?(:call) ? value.call : value
+    end
+  end
+end

data/lib/indexmap/index_now_configuration.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Indexmap
+  class IndexNowConfiguration
+    DEFAULT_ENDPOINT = "https://api.indexnow.org"
+    DEFAULT_MAX_URLS_PER_REQUEST = 500
+
+    attr_writer :dry_run, :endpoint, :key, :key_path, :max_urls_per_request
+
+    def dry_run?
+      value = resolve(@dry_run)
+      value == true || value.to_s == "1"
+    end
+
+    def endpoint
+      resolve(@endpoint).presence || DEFAULT_ENDPOINT
+    end
+
+    def key
+      resolve(@key)
+    end
+
+    def key_path(public_path:)
+      configured_path = resolve(@key_path)
+      return Pathname(configured_path) if configured_path.present?
+      return if key.to_s.strip.empty?
+
+      Pathname(public_path).join("#{key}.txt")
+    end
+
+    def max_urls_per_request
+      value = resolve(@max_urls_per_request)
+      return DEFAULT_MAX_URLS_PER_REQUEST if value.nil?
+
+      value.to_i
+    end
+
+    private
+
+    def resolve(value)
+      value.respond_to?(:call) ? value.call : value
+    end
+  end
+end

data/lib/indexmap/parser.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "nokogiri"
+require "pathname"
+require "uri"
+
+module Indexmap
+  class Parser
+    Entry = Struct.new(:loc, :lastmod, :source_sitemap, keyword_init: true)
+
+    def initialize(path: default_path, rebase_remote_children: false, index_filename: Indexmap.configuration.index_filename, public_path: Indexmap.configuration.public_path)
+      @source = path.to_s
+      @rebase_remote_children = rebase_remote_children
+      @index_filename = index_filename
+      @public_path = public_path
+    end
+
+    def entries(reset: false)
+      return reset! && entries if reset
+      return @entries if defined?(@entries)
+
+      visited = Set.new
+      @entries = parse_source(@source, visited: visited)
+    end
+
+    def paths(reset: false)
+      return reset! && paths if reset
+      return @paths if defined?(@paths)
+
+      seen = Set.new
+      @paths = entries.map do |entry|
+        path = extract_path(entry.loc)
+        next if path.nil?
+
+        normalized = normalize_path(path)
+        next if seen.include?(normalized)
+
+        seen.add(normalized)
+        normalized
+      end.compact
+    end
+
+    def urls(base_url:, reset: false)
+      return reset! && urls(base_url: base_url) if reset
+
+      target = URI.parse(base_url)
+      port_suffix = (target.port && ![80, 443].include?(target.port)) ? ":#{target.port}" : ""
+
+      paths.map do |path|
+        "#{target.scheme}://#{target.host}#{port_suffix}#{path}"
+      end
+    end
+
+    def reset!
+      remove_instance_variable(:@entries) if defined?(@entries)
+      remove_instance_variable(:@paths) if defined?(@paths)
+    end
+
+    private
+
+    attr_reader :index_filename, :public_path
+
+    def default_path
+      Indexmap::Path.existing_public_path(public_path: public_path, index_filename: index_filename)
+    end
+
+    def parse_source(source, visited:)
+      normalized_source = normalize_source(source)
+      return [] if normalized_source.nil? || visited.include?(normalized_source)
+
+      visited.add(normalized_source)
+      xml = read_source(normalized_source)
+      return [] if xml.to_s.strip.empty?
+
+      document = Nokogiri::XML(xml)
+      document.remove_namespaces!
+
+      if document.at_xpath("/sitemapindex")
+        document.xpath("//sitemap/loc").flat_map do |node|
+          child_source = resolve_child_sitemap(normalized_source, node.text.to_s.strip)
+          next [] if child_source.nil?
+
+          parse_source(child_source, visited: visited)
+        end
+      else
+        document.xpath("//url").map do |url_node|
+          loc = url_node.at_xpath("loc")&.text.to_s.strip
+          next if loc.empty?
+
+          lastmod = url_node.at_xpath("lastmod")&.text.to_s.strip
+          Entry.new(loc: loc, lastmod: lastmod.empty? ? nil : lastmod, source_sitemap: normalized_source)
+        end.compact
+      end
+    end
+
+    def resolve_child_sitemap(parent_source, loc)
+      return if loc.empty?
+
+      if remote_source?(parent_source)
+        parent_uri = URI.parse(parent_source)
+        if remote_source?(loc)
+          remote_child_source(parent_uri, loc)
+        else
+          URI.join(parent_uri.to_s, loc).to_s
+        end
+      elsif remote_source?(loc)
+        uri = URI.parse(loc)
+        File.join(File.dirname(parent_source), File.basename(uri.path))
+      else
+        File.expand_path(loc, File.dirname(parent_source))
+      end
+    rescue URI::InvalidURIError
+      File.expand_path(loc, File.dirname(parent_source))
+    end
+
+    def remote_child_source(parent_uri, loc)
+      child_uri = URI.parse(loc)
+      return child_uri.to_s unless @rebase_remote_children
+      return child_uri.to_s if child_uri.host == parent_uri.host && child_uri.port == parent_uri.port && child_uri.scheme == parent_uri.scheme
+
+      child_uri.scheme = parent_uri.scheme
+      child_uri.host = parent_uri.host
+      child_uri.port = parent_uri.port
+      child_uri.to_s
+    end
+
+    def normalize_source(source)
+      return if source.to_s.strip.empty?
+
+      if remote_source?(source)
+        URI.parse(source).to_s
+      else
+        Pathname(source).expand_path.to_s
+      end
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def remote_source?(value)
+      uri = URI.parse(value.to_s)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def read_source(source)
+      if remote_source?(source)
+        fetch_remote_source(source)
+      elsif File.exist?(source)
+        File.read(source, encoding: "UTF-8")
+      end
+    end
+
+    def fetch_remote_source(source, redirects_remaining: 3)
+      uri = URI.parse(source)
+      request = Net::HTTP::Get.new(uri)
+      request["User-Agent"] = "Indexmap::Parser/1.0"
+
+      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 10, read_timeout: 20) do |http|
+        http.request(request)
+      end
+
+      case response
+      when Net::HTTPSuccess
+        response.body
+      when Net::HTTPRedirection
+        return if redirects_remaining <= 0
+
+        location = response["location"].to_s
+        return if location.empty?
+
+        redirected = URI.join(source, location).to_s
+        fetch_remote_source(redirected, redirects_remaining: redirects_remaining - 1)
+      end
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def extract_path(loc)
+      return if loc.to_s.strip.empty?
+
+      if loc.start_with?("http://", "https://")
+        path = URI.parse(loc).path
+        (path.nil? || path.empty?) ? "/" : path
+      elsif loc.start_with?("/")
+        loc
+      else
+        "/#{loc}"
+      end
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def normalize_path(path)
+      return "/" if path == "/"
+
+      normalized = path.start_with?("/") ? path : "/#{path}"
+      normalized.delete_suffix("/")
+    end
+  end
+end

data/lib/indexmap/path.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "uri"
+
+module Indexmap
+  module Path
+    INDEX_FILENAME = "sitemap.xml"
+    LEGACY_FILENAME = "sitemap_index.xml"
+
+    module_function
+
+    def canonical_public_path(public_path: default_public_path_root, index_filename: default_index_filename)
+      Pathname(public_path).join(index_filename)
+    end
+
+    def existing_public_path(public_path: default_public_path_root, index_filename: default_index_filename, legacy_filename: LEGACY_FILENAME)
+      index_path = canonical_public_path(public_path: public_path, index_filename: index_filename)
+      return index_path if index_path.exist?
+
+      Pathname(public_path).join(legacy_filename)
+    end
+
+    def canonical_url(base_url, index_filename: default_index_filename)
+      URI.join(base_url, "/#{index_filename}").to_s
+    end
+
+    def default_index_filename
+      Indexmap.configuration.index_filename.presence || INDEX_FILENAME
+    rescue
+      INDEX_FILENAME
+    end
+
+    def default_public_path_root
+      if defined?(Rails)
+        Rails.public_path
+      else
+        Pathname("public")
+      end
+    end
+  end
+end

data/lib/indexmap/pinger/base.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "active_support/tagged_logging"
+require "uri"
+
+module Indexmap
+  module Pinger
+    class Base
+      def self.ping(...)
+        new(...).ping
+      end
+
+      def initialize(configuration: Indexmap.configuration)
+        @configuration = configuration
+      end
+
+      def ping
+        sitemap_files.each do |sitemap_file|
+          ping_sitemap(sitemap_file)
+        end
+      end
+
+      def logger
+        @logger ||= if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          Rails.logger
+        else
+          ActiveSupport::TaggedLogging.new(Logger.new($stdout))
+        end
+      end
+
+      private
+
+      attr_reader :configuration
+
+      def host
+        configuration.base_url
+      end
+
+      def hostname
+        URI.parse(host).host
+      end
+
+      def root_domain
+        hostname.sub(/\Awww\./, "")
+      end
+
+      def sitemap_files
+        Dir.glob(configuration.public_path.join("sitemap*.xml")).sort
+      end
+
+      def ping_sitemap(_sitemap_file)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/indexmap/pinger/google.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "google/apis/searchconsole_v1"
+require "googleauth"
+require "json"
+require "stringio"
+
+module Indexmap
+  module Pinger
+    class Google < Base
+      def initialize(configuration: Indexmap.configuration, service: nil, credentials_builder: nil)
+        super(configuration: configuration)
+        @service = service
+        @credentials_builder = credentials_builder
+      end
+
+      def ping
+        if google_configuration.credentials.to_s.strip.empty?
+          logger.debug("Google sitemap credentials not configured.")
+          return
+        end
+
+        super
+      end
+
+      private
+
+      attr_reader :credentials_builder
+
+      def google_configuration
+        configuration.google
+      end
+
+      def ping_sitemap(sitemap_file)
+        sitemap_url = URI.join(host, File.basename(sitemap_file)).to_s
+
+        unless authorized?
+          logger.error("Google Search Console does not have access to the site: #{root_domain}")
+          return
+        end
+
+        webmasters_service.submit_sitemap(property_identifier, sitemap_url)
+        logger.debug { "Successfully pinged Google with sitemap: #{sitemap_url}" }
+      rescue ::Google::Apis::ClientError => e
+        logger.debug { "Failed to ping Google for #{sitemap_url}. Status: #{e.status_code}, Body: #{e.body}" }
+      end
+
+      def authorized?
+        webmasters_service.list_sites.site_entry.any? { |site| site.site_url.include?(root_domain) }
+      end
+
+      def property_identifier
+        google_configuration.property.presence || "sc-domain:#{root_domain}"
+      end
+
+      def webmasters_service
+        @webmasters_service ||= begin
+          service = @service || ::Google::Apis::SearchconsoleV1::SearchConsoleService.new
+          service.authorization = authorizer
+          service
+        end
+      end
+
+      def authorizer
+        json_key = JSON.parse(google_configuration.credentials).to_json
+        scope = "https://www.googleapis.com/auth/webmasters"
+
+        return credentials_builder.call(credentials: json_key, scope: scope) if credentials_builder
+
+        ::Google::Auth::ServiceAccountCredentials.make_creds(
+          json_key_io: StringIO.new(json_key),
+          scope: scope
+        )
+      end
+    end
+  end
+end

data/lib/indexmap/pinger/index_now.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+require "time"
+
+module Indexmap
+  module Pinger
+    class IndexNow < Base
+      def initialize(configuration: Indexmap.configuration, connection: nil)
+        super(configuration: configuration)
+        @connection = connection
+      end
+
+      def ping
+        api_key = read_api_key
+        unless api_key
+          logger.debug("IndexNow API key is not configured.")
+          return
+        end
+
+        entries = entries_to_ping
+        if entries.empty?
+          logger.debug("IndexNow: no URLs matched the current filter.")
+          return
+        end
+
+        entries.each_slice(max_urls_per_request) do |batch|
+          urls = batch.map(&:loc)
+
+          if dry_run?
+            logger.debug { "IndexNow dry-run: would ping #{urls.count} URLs." }
+            next
+          end
+
+          submit_batch(api_key: api_key, urls: urls)
+        end
+      end
+
+      def write_key_file
+        key = index_now_configuration.key.to_s.strip
+        return if key.empty?
+
+        path = index_now_configuration.key_path(public_path: configuration.public_path)
+        FileUtils.mkdir_p(path.dirname)
+        File.write(path, "#{key}\n")
+        path
+      end
+
+      private
+
+      attr_reader :connection
+
+      def index_now_configuration
+        configuration.index_now
+      end
+
+      def sitemap_files
+        files = super
+        return files if files.one?
+
+        child_files = files.reject { |file| File.basename(file) == configuration.index_filename }
+        child_files.empty? ? files : child_files
+      end
+
+      def entries_to_ping
+        cutoff = since_cutoff
+        unless cutoff
+          logger.debug("IndexNow: no cutoff provided, submitting all sitemap URLs.")
+          return current_entries.values
+        end
+
+        logger.debug { "IndexNow: submitting sitemap URLs with lastmod >= #{cutoff.iso8601}." }
+
+        current_entries.values.select do |entry|
+          lastmod_after_cutoff?(entry, cutoff) || entry.lastmod.to_s.strip.empty?
+        end
+      end
+
+      def current_entries
+        sitemap_files.each_with_object({}) do |sitemap_file, entries|
+          Parser.new(path: sitemap_file).entries.each do |entry|
+            next if entry.loc.to_s.strip.empty?
+
+            entries[entry.loc] = entry
+          end
+        end
+      end
+
+      def since_cutoff
+        raw_value = ENV["SINCE"].to_s.strip
+        return recent_cutoff if raw_value.empty?
+
+        Time.iso8601(raw_value).utc
+      rescue ArgumentError
+        raise ArgumentError, "Invalid SINCE value: #{raw_value.inspect}. Use ISO 8601, e.g. 2026-04-18T10:30:00Z."
+      end
+
+      def recent_cutoff
+        hours = ENV["INDEXNOW_RECENT_HOURS"].to_s.strip
+        return if hours.empty?
+
+        hours_ago = Integer(hours, exception: false)
+        unless hours_ago&.positive?
+          raise ArgumentError, "Invalid INDEXNOW_RECENT_HOURS value: #{hours.inspect}. Use a positive integer."
+        end
+
+        Time.now.utc - (hours_ago * 3600)
+      end
+
+      def lastmod_after_cutoff?(entry, cutoff)
+        lastmod = entry_lastmod(entry)
+        return false unless lastmod
+
+        lastmod >= cutoff
+      end
+
+      def entry_lastmod(entry)
+        return if entry.lastmod.to_s.strip.empty?
+
+        Time.iso8601(entry.lastmod.to_s).utc
+      rescue ArgumentError
+        logger.debug { "IndexNow: skipping invalid sitemap lastmod #{entry.lastmod.inspect} for #{entry.loc}" }
+        nil
+      end
+
+      def max_urls_per_request
+        ENV.fetch("INDEXNOW_MAX_URLS_PER_REQUEST", index_now_configuration.max_urls_per_request).to_i
+      end
+
+      def submit_batch(api_key:, urls:)
+        payload = {host: hostname, key: api_key, urlList: urls}
+        response = index_now_connection.post("/indexnow") do |request|
+          request.headers["Content-Type"] = "application/json"
+          request.body = payload.to_json
+        end
+
+        if response.success?
+          logger.debug { "Successfully pinged IndexNow with #{urls.count} URLs." }
+          true
+        else
+          logger.debug { "Failed to ping IndexNow. Status: #{response.status}, Body: #{response.body}" }
+          false
+        end
+      end
+
+      def index_now_connection
+        @index_now_connection ||= connection || Faraday.new(url: index_now_configuration.endpoint) do |faraday|
+          faraday.request :json
+        end
+      end
+
+      def dry_run?
+        ENV["INDEXNOW_DRY_RUN"] == "1" || index_now_configuration.dry_run?
+      end
+
+      def read_api_key
+        configured_key = index_now_configuration.key.to_s.strip
+        return configured_key unless configured_key.empty?
+
+        key_file = configuration.public_path.glob("*.txt").find do |file|
+          filename = file.basename(".txt").to_s
+          next unless filename.match?(/\A[a-zA-Z0-9-]{8,128}\z/)
+
+          File.read(file).strip == filename
+        end
+        return nil unless key_file
+
+        File.read(key_file).strip
+      end
+    end
+  end
+end

data/lib/indexmap/task_runner.rb

@@ -11,6 +11,7 @@ module Indexmap
     def create
       remove_existing_sitemap_files
       configuration.writer.write
+      write_index_now_key
     end
 
     def format
@@ -28,6 +29,14 @@ module Indexmap
       end
     end
 
+    def validate
+      Validator.new(configuration: configuration).validate!
+    end
+
+    def write_index_now_key
+      Pinger::IndexNow.new(configuration: configuration).write_key_file
+    end
+
     private
 
     attr_reader :configuration

data/lib/indexmap/validator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Indexmap
+  class Validator
+    def initialize(configuration: Indexmap.configuration, path: nil)
+      @configuration = configuration
+      @path = path
+    end
+
+    def validate!
+      sitemap_path = path || Indexmap::Path.existing_public_path(
+        public_path: configuration.public_path,
+        index_filename: configuration.index_filename
+      )
+      raise ValidationError, "Missing sitemap file: #{sitemap_path}" unless File.exist?(sitemap_path)
+
+      entries = Parser.new(path: sitemap_path).entries
+      validate_duplicates!(entries)
+      validate_parameterized_urls!(entries)
+      true
+    end
+
+    private
+
+    attr_reader :configuration, :path
+
+    def validate_duplicates!(entries)
+      duplicates = entries.map(&:loc).group_by(&:itself).select { |_url, values| values.size > 1 }.keys
+      return if duplicates.empty?
+
+      raise ValidationError, "Duplicate sitemap URLs detected: #{duplicates.first(5).join(", ")}"
+    end
+
+    def validate_parameterized_urls!(entries)
+      param_urls = entries.map(&:loc).select { |url| url&.include?("?") }
+      return if param_urls.empty?
+
+      raise ValidationError, "Parameterized sitemap URLs detected: #{param_urls.first(5).join(", ")}"
+    end
+  end
+end

data/lib/indexmap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Indexmap
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/indexmap.rb

@@ -7,16 +7,25 @@ require "pathname"
 require "time"
 
 require_relative "indexmap/version"
+require_relative "indexmap/google_configuration"
+require_relative "indexmap/index_now_configuration"
 require_relative "indexmap/configuration"
 require_relative "indexmap/entry"
+require_relative "indexmap/path"
+require_relative "indexmap/parser"
+require_relative "indexmap/pinger/base"
+require_relative "indexmap/pinger/google"
+require_relative "indexmap/pinger/index_now"
 require_relative "indexmap/section"
 require_relative "indexmap/task_runner"
+require_relative "indexmap/validator"
 require_relative "indexmap/writer"
 
 module Indexmap
   class Error < StandardError; end
 
   class ConfigurationError < Error; end
+  class ValidationError < Error; end
 
   class << self
     def configuration

data/lib/tasks/indexmap_tasks.rake

@@ -4,10 +4,42 @@ namespace :sitemap do
     runner = Indexmap::TaskRunner.new
     runner.create
     runner.format
+    runner.validate
   end
 
   desc "Format sitemap files for better readability"
   task format: :environment do
     Indexmap::TaskRunner.new.format
   end
+
+  desc "Validate sitemap shape and URL hygiene"
+  task validate: :environment do
+    Indexmap::TaskRunner.new.validate
+  end
+
+  desc "Ping all configured search engines"
+  task ping: :environment do
+    Rake::Task["sitemap:index_now:ping"].invoke
+    Rake::Task["sitemap:google:ping"].invoke
+  end
+
+  namespace :google do
+    desc "Ping Google Search Console"
+    task ping: :environment do
+      Indexmap::Pinger::Google.new.ping
+    end
+  end
+
+  namespace :index_now do
+    desc "Ping IndexNow. ENV: SINCE=2026-04-18T10:30:00Z or INDEXNOW_RECENT_HOURS=24"
+    task ping: :environment do
+      Indexmap::Pinger::IndexNow.new.ping
+    end
+
+    desc "Write the IndexNow key file into public/"
+    task write_key: :environment do
+      path = Indexmap::TaskRunner.new.write_index_now_key
+      puts "Wrote #{path}" if path
+    end
+  end
 end

data/test/indexmap/configuration_test.rb

@@ -66,4 +66,18 @@ class IndexmapConfigurationTest < Minitest::Test
 
     assert_equal "Indexmap format must be one of: index, single_file", error.message
   end
+
+  def test_exposes_nested_google_and_index_now_configuration
+    Indexmap.configure do |config|
+      config.google.credentials = -> { "{\"type\":\"service_account\"}" }
+      config.google.property = -> { "sc-domain:example.com" }
+      config.index_now.key = -> { "example-key" }
+      config.index_now.max_urls_per_request = -> { 250 }
+    end
+
+    assert_equal "{\"type\":\"service_account\"}", Indexmap.configuration.google.credentials
+    assert_equal "sc-domain:example.com", Indexmap.configuration.google.property
+    assert_equal "example-key", Indexmap.configuration.index_now.key
+    assert_equal 250, Indexmap.configuration.index_now.max_urls_per_request
+  end
 end

data/test/indexmap/parser_test.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapParserTest < Minitest::Test
+  def test_parses_remote_sitemap_urlset
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.example.com/</loc></url>
+            <url><loc>https://www.example.com/pages/features</loc></url>
+          </urlset>
+        XML
+      )
+
+    parser = Indexmap::Parser.new(path: "https://www.example.com/sitemap.xml")
+
+    assert_equal ["/", "/pages/features"], parser.paths
+  end
+
+  def test_parses_remote_sitemap_index_with_child_sitemap
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>/sitemaps/content.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+
+    stub_request(:get, "https://www.example.com/sitemaps/content.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.example.com/tools/google-reviews-calculator</loc></url>
+          </urlset>
+        XML
+      )
+
+    parser = Indexmap::Parser.new(path: "https://www.example.com/sitemap.xml")
+
+    assert_equal ["/tools/google-reviews-calculator"], parser.paths
+    assert_equal ["https://www.reviato.com/tools/google-reviews-calculator"], parser.urls(base_url: "https://www.reviato.com")
+  end
+
+  def test_can_rebase_remote_child_sitemap_urls_to_the_fetched_sitemap_origin
+    stub_request(:get, "http://localhost:3001/sitemap.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>https://www.reviato.com/sitemap-marketing.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+
+    stub_request(:get, "http://localhost:3001/sitemap-marketing.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.reviato.com/pages/pricing</loc></url>
+          </urlset>
+        XML
+      )
+
+    parser = Indexmap::Parser.new(path: "http://localhost:3001/sitemap.xml", rebase_remote_children: true)
+
+    assert_equal ["/pages/pricing"], parser.paths
+    assert_equal ["http://localhost:3001/pages/pricing"], parser.urls(base_url: "http://localhost:3001")
+  end
+end

data/test/indexmap/path_test.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapPathTest < Minitest::Test
+  def test_existing_public_path_prefers_sitemap_index_when_present
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap_index.xml").write("<urlset/>")
+      public_path.join("sitemap.xml").write("<sitemapindex/>")
+
+      assert_equal public_path.join("sitemap.xml"), Indexmap::Path.existing_public_path(public_path: public_path)
+    end
+  end
+
+  def test_existing_public_path_falls_back_to_legacy_sitemap_path
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap_index.xml").write("<sitemapindex/>")
+
+      assert_equal public_path.join("sitemap_index.xml"), Indexmap::Path.existing_public_path(public_path: public_path)
+    end
+  end
+
+  def test_canonical_url_targets_sitemap_index
+    assert_equal "https://www.example.com/sitemap.xml", Indexmap::Path.canonical_url("https://www.example.com")
+  end
+end

data/test/indexmap/pinger/google_test.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapPingerGoogleTest < Minitest::Test
+  SiteEntry = Struct.new(:site_url)
+  SiteList = Struct.new(:site_entry)
+
+  class FakeWebmastersService
+    attr_accessor :authorization
+    attr_reader :submitted
+
+    def initialize(site_urls:)
+      @site_urls = site_urls
+    end
+
+    def list_sites
+      SiteList.new(@site_urls.map { |site_url| SiteEntry.new(site_url) })
+    end
+
+    def submit_sitemap(property, sitemap_url)
+      @submitted = [property, sitemap_url]
+    end
+  end
+
+  def test_pings_google_for_each_sitemap_file
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap.xml").write("<sitemapindex/>")
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+      configuration.google.credentials = "{\"type\":\"service_account\"}"
+
+      service = FakeWebmastersService.new(site_urls: ["sc-domain:example.com"])
+      builder_calls = []
+      credentials_builder = lambda do |credentials:, scope:|
+        builder_calls << [credentials, scope]
+        :fake_authorizer
+      end
+
+      Indexmap::Pinger::Google.new(
+        configuration: configuration,
+        service: service,
+        credentials_builder: credentials_builder
+      ).ping
+
+      assert_equal [["{\"type\":\"service_account\"}", "https://www.googleapis.com/auth/webmasters"]], builder_calls
+      assert_equal :fake_authorizer, service.authorization
+      assert_equal ["sc-domain:example.com", "https://www.example.com/sitemap.xml"], service.submitted
+    end
+  end
+
+  def test_skips_google_ping_when_credentials_are_missing
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap.xml").write("<sitemapindex/>")
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+
+      service = FakeWebmastersService.new(site_urls: ["sc-domain:example.com"])
+
+      Indexmap::Pinger::Google.new(configuration: configuration, service: service).ping
+
+      assert_nil service.submitted
+    end
+  end
+end

data/test/indexmap/pinger/index_now_test.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapPingerIndexNowTest < Minitest::Test
+  def test_writes_key_file_from_configuration
+    Dir.mktmpdir do |dir|
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = Pathname(dir)
+      configuration.index_now.key = "test-key"
+
+      path = Indexmap::Pinger::IndexNow.new(configuration: configuration).write_key_file
+
+      assert_equal Pathname(dir).join("test-key.txt"), path
+      assert_equal "test-key\n", path.read
+    end
+  end
+
+  def test_pings_all_sitemap_urls_when_no_cutoff_is_provided
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      write_sitemap_files(
+        public_path,
+        marketing_lastmod: "2026-04-18T00:00:00Z",
+        insights_lastmod: "2026-04-10T00:00:00Z"
+      )
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+      configuration.index_now.key = "test-key"
+
+      indexnow_url = "https://api.indexnow.org/indexnow"
+      stub_request(:post, indexnow_url).to_return(status: 200, body: "", headers: {})
+
+      Indexmap::Pinger::IndexNow.new(configuration: configuration).ping
+
+      assert_requested(:post, indexnow_url, times: 1) do |request|
+        payload = JSON.parse(request.body)
+        assert_equal [
+          "https://www.example.com/pages/features",
+          "https://www.example.com/insights/us/restaurants/overview"
+        ].sort, payload.fetch("urlList").sort
+      end
+    end
+  end
+
+  def test_pings_only_sitemap_urls_newer_than_since
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      write_sitemap_files(
+        public_path,
+        marketing_lastmod: "2026-04-18T00:00:00Z",
+        insights_lastmod: "2026-04-10T00:00:00Z"
+      )
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+      configuration.index_now.key = "test-key"
+
+      indexnow_url = "https://api.indexnow.org/indexnow"
+      stub_request(:post, indexnow_url).to_return(status: 200, body: "", headers: {})
+
+      with_env("SINCE" => "2026-04-15T00:00:00Z") do
+        Indexmap::Pinger::IndexNow.new(configuration: configuration).ping
+      end
+
+      assert_requested(:post, indexnow_url, times: 1) do |request|
+        payload = JSON.parse(request.body)
+        assert_equal ["https://www.example.com/pages/features"], payload.fetch("urlList")
+      end
+    end
+  end
+
+  private
+
+  def with_env(overrides)
+    previous_values = overrides.to_h { |key, _value| [key, ENV[key]] }
+    overrides.each { |key, value| ENV[key] = value }
+    yield
+  ensure
+    previous_values.each do |key, value|
+      value.nil? ? ENV.delete(key) : ENV[key] = value
+    end
+  end
+
+  def write_sitemap_files(public_path, marketing_lastmod:, insights_lastmod:)
+    public_path.join("sitemap.xml").write(<<~XML)
+      <?xml version="1.0" encoding="UTF-8"?>
+      <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <sitemap><loc>https://www.example.com/sitemap-marketing.xml</loc></sitemap>
+        <sitemap><loc>https://www.example.com/sitemap-insights.xml</loc></sitemap>
+      </sitemapindex>
+    XML
+
+    public_path.join("sitemap-marketing.xml").write(<<~XML)
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>https://www.example.com/pages/features</loc>
+          <lastmod>#{marketing_lastmod}</lastmod>
+        </url>
+      </urlset>
+    XML
+
+    public_path.join("sitemap-insights.xml").write(<<~XML)
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>https://www.example.com/insights/us/restaurants/overview</loc>
+          <lastmod>#{insights_lastmod}</lastmod>
+        </url>
+      </urlset>
+    XML
+  end
+end

data/test/indexmap/validator_test.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapValidatorTest < Minitest::Test
+  def test_validate_raises_for_duplicate_urls
+    Dir.mktmpdir do |directory|
+      path = Pathname(directory).join("sitemap.xml")
+      path.write(<<~XML)
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/about</loc></url>
+          <url><loc>https://example.com/about</loc></url>
+        </urlset>
+      XML
+
+      error = assert_raises(Indexmap::ValidationError) do
+        Indexmap::Validator.new(path: path).validate!
+      end
+
+      assert_equal "Duplicate sitemap URLs detected: https://example.com/about", error.message
+    end
+  end
+
+  def test_validate_raises_for_parameterized_urls
+    Dir.mktmpdir do |directory|
+      path = Pathname(directory).join("sitemap.xml")
+      path.write(<<~XML)
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/about?ref=test</loc></url>
+        </urlset>
+      XML
+
+      error = assert_raises(Indexmap::ValidationError) do
+        Indexmap::Validator.new(path: path).validate!
+      end
+
+      assert_equal "Parameterized sitemap URLs detected: https://example.com/about?ref=test", error.message
+    end
+  end
+
+  def test_validate_passes_for_valid_sitemap
+    Dir.mktmpdir do |directory|
+      path = Pathname(directory).join("sitemap.xml")
+      path.write(<<~XML)
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/about</loc></url>
+        </urlset>
+      XML
+
+      assert Indexmap::Validator.new(path: path).validate!
+    end
+  end
+end

data/test/test_helper.rb

@@ -5,5 +5,6 @@ $LOAD_PATH.unshift File.expand_path("../lib", __dir__)
 require "minitest/autorun"
 require "tmpdir"
 require "date"
+require "webmock/minitest"
 
 require "indexmap"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexmap
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -24,6 +24,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: google-apis-searchconsole_v1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: googleauth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.12'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 description: A small Ruby gem for generating sitemap indexes and child sitemaps from
   explicit section definitions, with optional Rails rake task integration.
 email:
@@ -108,13 +164,26 @@ files:
 - lib/indexmap.rb
 - lib/indexmap/configuration.rb
 - lib/indexmap/entry.rb
+- lib/indexmap/google_configuration.rb
+- lib/indexmap/index_now_configuration.rb
+- lib/indexmap/parser.rb
+- lib/indexmap/path.rb
+- lib/indexmap/pinger/base.rb
+- lib/indexmap/pinger/google.rb
+- lib/indexmap/pinger/index_now.rb
 - lib/indexmap/railtie.rb
 - lib/indexmap/section.rb
 - lib/indexmap/task_runner.rb
+- lib/indexmap/validator.rb
 - lib/indexmap/version.rb
 - lib/indexmap/writer.rb
 - lib/tasks/indexmap_tasks.rake
 - test/indexmap/configuration_test.rb
+- test/indexmap/parser_test.rb
+- test/indexmap/path_test.rb
+- test/indexmap/pinger/google_test.rb
+- test/indexmap/pinger/index_now_test.rb
+- test/indexmap/validator_test.rb
 - test/indexmap/writer_test.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/indexmap