indexmap 0.2.1 → 0.3.1

data/lib/indexmap/pinger/index_now.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+require "time"
+
+module Indexmap
+  module Pinger
+    class IndexNow < Base
+      def initialize(configuration: Indexmap.configuration, connection: nil)
+        super(configuration: configuration)
+        @connection = connection
+      end
+
+      def ping
+        api_key = read_api_key
+        unless api_key
+          logger.debug("IndexNow API key is not configured.")
+          return
+        end
+
+        entries = entries_to_ping
+        if entries.empty?
+          logger.debug("IndexNow: no URLs matched the current filter.")
+          return
+        end
+
+        entries.each_slice(max_urls_per_request) do |batch|
+          urls = batch.map(&:loc)
+
+          if dry_run?
+            logger.debug { "IndexNow dry-run: would ping #{urls.count} URLs." }
+            next
+          end
+
+          submit_batch(api_key: api_key, urls: urls)
+        end
+      end
+
+      def write_key_file
+        key = index_now_configuration.key.to_s.strip
+        return if key.empty?
+
+        path = index_now_configuration.key_path(public_path: configuration.public_path)
+        FileUtils.mkdir_p(path.dirname)
+        File.write(path, "#{key}\n")
+        path
+      end
+
+      private
+
+      attr_reader :connection
+
+      def index_now_configuration
+        configuration.index_now
+      end
+
+      def sitemap_files
+        files = super
+        return files if files.one?
+
+        child_files = files.reject { |file| File.basename(file) == configuration.index_filename }
+        child_files.empty? ? files : child_files
+      end
+
+      def entries_to_ping
+        cutoff = since_cutoff
+        unless cutoff
+          logger.debug("IndexNow: no cutoff provided, submitting all sitemap URLs.")
+          return current_entries.values
+        end
+
+        logger.debug { "IndexNow: submitting sitemap URLs with lastmod >= #{cutoff.iso8601}." }
+
+        current_entries.values.select do |entry|
+          lastmod_after_cutoff?(entry, cutoff) || entry.lastmod.to_s.strip.empty?
+        end
+      end
+
+      def current_entries
+        sitemap_files.each_with_object({}) do |sitemap_file, entries|
+          Parser.new(path: sitemap_file).entries.each do |entry|
+            next if entry.loc.to_s.strip.empty?
+
+            entries[entry.loc] = entry
+          end
+        end
+      end
+
+      def since_cutoff
+        raw_value = ENV["SINCE"].to_s.strip
+        return recent_cutoff if raw_value.empty?
+
+        Time.iso8601(raw_value).utc
+      rescue ArgumentError
+        raise ArgumentError, "Invalid SINCE value: #{raw_value.inspect}. Use ISO 8601, e.g. 2026-04-18T10:30:00Z."
+      end
+
+      def recent_cutoff
+        hours = ENV["INDEXNOW_RECENT_HOURS"].to_s.strip
+        return if hours.empty?
+
+        hours_ago = Integer(hours, exception: false)
+        unless hours_ago&.positive?
+          raise ArgumentError, "Invalid INDEXNOW_RECENT_HOURS value: #{hours.inspect}. Use a positive integer."
+        end
+
+        Time.now.utc - (hours_ago * 3600)
+      end
+
+      def lastmod_after_cutoff?(entry, cutoff)
+        lastmod = entry_lastmod(entry)
+        return false unless lastmod
+
+        lastmod >= cutoff
+      end
+
+      def entry_lastmod(entry)
+        return if entry.lastmod.to_s.strip.empty?
+
+        Time.iso8601(entry.lastmod.to_s).utc
+      rescue ArgumentError
+        logger.debug { "IndexNow: skipping invalid sitemap lastmod #{entry.lastmod.inspect} for #{entry.loc}" }
+        nil
+      end
+
+      def max_urls_per_request
+        ENV.fetch("INDEXNOW_MAX_URLS_PER_REQUEST", index_now_configuration.max_urls_per_request).to_i
+      end
+
+      def submit_batch(api_key:, urls:)
+        payload = {host: hostname, key: api_key, urlList: urls}
+        response = index_now_connection.post("/indexnow") do |request|
+          request.headers["Content-Type"] = "application/json"
+          request.body = payload.to_json
+        end
+
+        if response.success?
+          logger.debug { "Successfully pinged IndexNow with #{urls.count} URLs." }
+          true
+        else
+          logger.debug { "Failed to ping IndexNow. Status: #{response.status}, Body: #{response.body}" }
+          false
+        end
+      end
+
+      def index_now_connection
+        @index_now_connection ||= connection || Faraday.new(url: index_now_configuration.endpoint) do |faraday|
+          faraday.request :json
+        end
+      end
+
+      def dry_run?
+        ENV["INDEXNOW_DRY_RUN"] == "1" || index_now_configuration.dry_run?
+      end
+
+      def read_api_key
+        configured_key = index_now_configuration.key.to_s.strip
+        return configured_key unless configured_key.empty?
+
+        key_file = configuration.public_path.glob("*.txt").find do |file|
+          filename = file.basename(".txt").to_s
+          next unless filename.match?(/\A[a-zA-Z0-9-]{8,128}\z/)
+
+          File.read(file).strip == filename
+        end
+        return nil unless key_file
+
+        File.read(key_file).strip
+      end
+    end
+  end
+end

data/lib/indexmap/task_runner.rb

@@ -11,6 +11,7 @@ module Indexmap
     def create
       remove_existing_sitemap_files
       configuration.writer.write
+      write_index_now_key
     end
 
     def format
@@ -28,6 +29,14 @@ module Indexmap
       end
     end
 
+    def validate
+      Validator.new(configuration: configuration).validate!
+    end
+
+    def write_index_now_key
+      Pinger::IndexNow.new(configuration: configuration).write_key_file
+    end
+
     private
 
     attr_reader :configuration

data/lib/indexmap/validator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Indexmap
+  class Validator
+    def initialize(configuration: Indexmap.configuration, path: nil)
+      @configuration = configuration
+      @path = path
+    end
+
+    def validate!
+      sitemap_path = path || Indexmap::Path.existing_public_path(
+        public_path: configuration.public_path,
+        index_filename: configuration.index_filename
+      )
+      raise ValidationError, "Missing sitemap file: #{sitemap_path}" unless File.exist?(sitemap_path)
+
+      entries = Parser.new(path: sitemap_path).entries
+      validate_duplicates!(entries)
+      validate_parameterized_urls!(entries)
+      true
+    end
+
+    private
+
+    attr_reader :configuration, :path
+
+    def validate_duplicates!(entries)
+      duplicates = entries.map(&:loc).group_by(&:itself).select { |_url, values| values.size > 1 }.keys
+      return if duplicates.empty?
+
+      raise ValidationError, "Duplicate sitemap URLs detected: #{duplicates.first(5).join(", ")}"
+    end
+
+    def validate_parameterized_urls!(entries)
+      param_urls = entries.map(&:loc).select { |url| url&.include?("?") }
+      return if param_urls.empty?
+
+      raise ValidationError, "Parameterized sitemap URLs detected: #{param_urls.first(5).join(", ")}"
+    end
+  end
+end

data/lib/indexmap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Indexmap
-  VERSION = "0.2.1"
+  VERSION = "0.3.1"
 end

data/lib/indexmap.rb

@@ -7,16 +7,25 @@ require "pathname"
 require "time"
 
 require_relative "indexmap/version"
+require_relative "indexmap/google_configuration"
+require_relative "indexmap/index_now_configuration"
 require_relative "indexmap/configuration"
 require_relative "indexmap/entry"
+require_relative "indexmap/path"
+require_relative "indexmap/parser"
+require_relative "indexmap/pinger/base"
+require_relative "indexmap/pinger/google"
+require_relative "indexmap/pinger/index_now"
 require_relative "indexmap/section"
 require_relative "indexmap/task_runner"
+require_relative "indexmap/validator"
 require_relative "indexmap/writer"
 
 module Indexmap
   class Error < StandardError; end
 
   class ConfigurationError < Error; end
+  class ValidationError < Error; end
 
   class << self
     def configuration

data/lib/tasks/indexmap_tasks.rake

@@ -4,10 +4,42 @@ namespace :sitemap do
     runner = Indexmap::TaskRunner.new
     runner.create
     runner.format
+    runner.validate
   end
 
   desc "Format sitemap files for better readability"
   task format: :environment do
     Indexmap::TaskRunner.new.format
   end
+
+  desc "Validate sitemap shape and URL hygiene"
+  task validate: :environment do
+    Indexmap::TaskRunner.new.validate
+  end
+
+  desc "Ping all configured search engines"
+  task ping: :environment do
+    Rake::Task["sitemap:index_now:ping"].invoke
+    Rake::Task["sitemap:google:ping"].invoke
+  end
+
+  namespace :google do
+    desc "Ping Google Search Console"
+    task ping: :environment do
+      Indexmap::Pinger::Google.new.ping
+    end
+  end
+
+  namespace :index_now do
+    desc "Ping IndexNow. ENV: SINCE=2026-04-18T10:30:00Z or INDEXNOW_RECENT_HOURS=24"
+    task ping: :environment do
+      Indexmap::Pinger::IndexNow.new.ping
+    end
+
+    desc "Write the IndexNow key file into public/"
+    task write_key: :environment do
+      path = Indexmap::TaskRunner.new.write_index_now_key
+      puts "Wrote #{path}" if path
+    end
+  end
 end

data/test/indexmap/configuration_test.rb

@@ -66,4 +66,18 @@ class IndexmapConfigurationTest < Minitest::Test
 
     assert_equal "Indexmap format must be one of: index, single_file", error.message
   end
+
+  def test_exposes_nested_google_and_index_now_configuration
+    Indexmap.configure do |config|
+      config.google.credentials = -> { "{\"type\":\"service_account\"}" }
+      config.google.property = -> { "sc-domain:example.com" }
+      config.index_now.key = -> { "example-key" }
+      config.index_now.max_urls_per_request = -> { 250 }
+    end
+
+    assert_equal "{\"type\":\"service_account\"}", Indexmap.configuration.google.credentials
+    assert_equal "sc-domain:example.com", Indexmap.configuration.google.property
+    assert_equal "example-key", Indexmap.configuration.index_now.key
+    assert_equal 250, Indexmap.configuration.index_now.max_urls_per_request
+  end
 end

data/test/indexmap/parser_test.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapParserTest < Minitest::Test
+  def test_parses_remote_sitemap_urlset
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.example.com/</loc></url>
+            <url><loc>https://www.example.com/pages/features</loc></url>
+          </urlset>
+        XML
+      )
+
+    parser = Indexmap::Parser.new(path: "https://www.example.com/sitemap.xml")
+
+    assert_equal ["/", "/pages/features"], parser.paths
+  end
+
+  def test_parses_remote_sitemap_index_with_child_sitemap
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>/sitemaps/content.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+
+    stub_request(:get, "https://www.example.com/sitemaps/content.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.example.com/tools/google-reviews-calculator</loc></url>
+          </urlset>
+        XML
+      )
+
+    parser = Indexmap::Parser.new(path: "https://www.example.com/sitemap.xml")
+
+    assert_equal ["/tools/google-reviews-calculator"], parser.paths
+    assert_equal ["https://www.reviato.com/tools/google-reviews-calculator"], parser.urls(base_url: "https://www.reviato.com")
+  end
+
+  def test_can_rebase_remote_child_sitemap_urls_to_the_fetched_sitemap_origin
+    stub_request(:get, "http://localhost:3001/sitemap.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>https://www.reviato.com/sitemap-marketing.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+
+    stub_request(:get, "http://localhost:3001/sitemap-marketing.xml")
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.reviato.com/pages/pricing</loc></url>
+          </urlset>
+        XML
+      )
+
+    parser = Indexmap::Parser.new(path: "http://localhost:3001/sitemap.xml", rebase_remote_children: true)
+
+    assert_equal ["/pages/pricing"], parser.paths
+    assert_equal ["http://localhost:3001/pages/pricing"], parser.urls(base_url: "http://localhost:3001")
+  end
+end

data/test/indexmap/path_test.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapPathTest < Minitest::Test
+  def test_existing_public_path_prefers_sitemap_index_when_present
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap_index.xml").write("<urlset/>")
+      public_path.join("sitemap.xml").write("<sitemapindex/>")
+
+      assert_equal public_path.join("sitemap.xml"), Indexmap::Path.existing_public_path(public_path: public_path)
+    end
+  end
+
+  def test_existing_public_path_falls_back_to_legacy_sitemap_path
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap_index.xml").write("<sitemapindex/>")
+
+      assert_equal public_path.join("sitemap_index.xml"), Indexmap::Path.existing_public_path(public_path: public_path)
+    end
+  end
+
+  def test_canonical_url_targets_sitemap_index
+    assert_equal "https://www.example.com/sitemap.xml", Indexmap::Path.canonical_url("https://www.example.com")
+  end
+end

data/test/indexmap/pinger/google_test.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapPingerGoogleTest < Minitest::Test
+  SiteEntry = Struct.new(:site_url)
+  SiteList = Struct.new(:site_entry)
+
+  class FakeWebmastersService
+    attr_accessor :authorization
+    attr_reader :submitted
+
+    def initialize(site_urls:)
+      @site_urls = site_urls
+    end
+
+    def list_sites
+      SiteList.new(@site_urls.map { |site_url| SiteEntry.new(site_url) })
+    end
+
+    def submit_sitemap(property, sitemap_url)
+      @submitted = [property, sitemap_url]
+    end
+  end
+
+  def test_pings_google_for_each_sitemap_file
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap.xml").write("<sitemapindex/>")
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+      configuration.google.credentials = "{\"type\":\"service_account\"}"
+
+      service = FakeWebmastersService.new(site_urls: ["sc-domain:example.com"])
+      builder_calls = []
+      credentials_builder = lambda do |credentials:, scope:|
+        builder_calls << [credentials, scope]
+        :fake_authorizer
+      end
+
+      Indexmap::Pinger::Google.new(
+        configuration: configuration,
+        service: service,
+        credentials_builder: credentials_builder
+      ).ping
+
+      assert_equal [["{\"type\":\"service_account\"}", "https://www.googleapis.com/auth/webmasters"]], builder_calls
+      assert_equal :fake_authorizer, service.authorization
+      assert_equal ["sc-domain:example.com", "https://www.example.com/sitemap.xml"], service.submitted
+    end
+  end
+
+  def test_skips_google_ping_when_credentials_are_missing
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap.xml").write("<sitemapindex/>")
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+
+      service = FakeWebmastersService.new(site_urls: ["sc-domain:example.com"])
+
+      Indexmap::Pinger::Google.new(configuration: configuration, service: service).ping
+
+      assert_nil service.submitted
+    end
+  end
+end

data/test/indexmap/pinger/index_now_test.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapPingerIndexNowTest < Minitest::Test
+  def test_writes_key_file_from_configuration
+    Dir.mktmpdir do |dir|
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = Pathname(dir)
+      configuration.index_now.key = "test-key"
+
+      path = Indexmap::Pinger::IndexNow.new(configuration: configuration).write_key_file
+
+      assert_equal Pathname(dir).join("test-key.txt"), path
+      assert_equal "test-key\n", path.read
+    end
+  end
+
+  def test_pings_all_sitemap_urls_when_no_cutoff_is_provided
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      write_sitemap_files(
+        public_path,
+        marketing_lastmod: "2026-04-18T00:00:00Z",
+        insights_lastmod: "2026-04-10T00:00:00Z"
+      )
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+      configuration.index_now.key = "test-key"
+
+      indexnow_url = "https://api.indexnow.org/indexnow"
+      stub_request(:post, indexnow_url).to_return(status: 200, body: "", headers: {})
+
+      Indexmap::Pinger::IndexNow.new(configuration: configuration).ping
+
+      assert_requested(:post, indexnow_url, times: 1) do |request|
+        payload = JSON.parse(request.body)
+        assert_equal [
+          "https://www.example.com/pages/features",
+          "https://www.example.com/insights/us/restaurants/overview"
+        ].sort, payload.fetch("urlList").sort
+      end
+    end
+  end
+
+  def test_pings_only_sitemap_urls_newer_than_since
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      write_sitemap_files(
+        public_path,
+        marketing_lastmod: "2026-04-18T00:00:00Z",
+        insights_lastmod: "2026-04-10T00:00:00Z"
+      )
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://www.example.com"
+      configuration.public_path = public_path
+      configuration.index_now.key = "test-key"
+
+      indexnow_url = "https://api.indexnow.org/indexnow"
+      stub_request(:post, indexnow_url).to_return(status: 200, body: "", headers: {})
+
+      with_env("SINCE" => "2026-04-15T00:00:00Z") do
+        Indexmap::Pinger::IndexNow.new(configuration: configuration).ping
+      end
+
+      assert_requested(:post, indexnow_url, times: 1) do |request|
+        payload = JSON.parse(request.body)
+        assert_equal ["https://www.example.com/pages/features"], payload.fetch("urlList")
+      end
+    end
+  end
+
+  private
+
+  def with_env(overrides)
+    previous_values = overrides.to_h { |key, _value| [key, ENV[key]] }
+    overrides.each { |key, value| ENV[key] = value }
+    yield
+  ensure
+    previous_values.each do |key, value|
+      value.nil? ? ENV.delete(key) : ENV[key] = value
+    end
+  end
+
+  def write_sitemap_files(public_path, marketing_lastmod:, insights_lastmod:)
+    public_path.join("sitemap.xml").write(<<~XML)
+      <?xml version="1.0" encoding="UTF-8"?>
+      <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <sitemap><loc>https://www.example.com/sitemap-marketing.xml</loc></sitemap>
+        <sitemap><loc>https://www.example.com/sitemap-insights.xml</loc></sitemap>
+      </sitemapindex>
+    XML
+
+    public_path.join("sitemap-marketing.xml").write(<<~XML)
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>https://www.example.com/pages/features</loc>
+          <lastmod>#{marketing_lastmod}</lastmod>
+        </url>
+      </urlset>
+    XML
+
+    public_path.join("sitemap-insights.xml").write(<<~XML)
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>https://www.example.com/insights/us/restaurants/overview</loc>
+          <lastmod>#{insights_lastmod}</lastmod>
+        </url>
+      </urlset>
+    XML
+  end
+end

data/test/indexmap/task_runner_test.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class IndexmapTaskRunnerTest < Minitest::Test
+  def test_create_removes_existing_sitemap_files_writes_new_sitemap_and_key_file
+    Dir.mktmpdir do |dir|
+      public_path = Pathname(dir)
+      public_path.join("sitemap.xml").write("old")
+      public_path.join("sitemap-pages.xml.gz").write("old")
+
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://example.com"
+      configuration.public_path = public_path
+      configuration.sections = [
+        Indexmap::Section.new(
+          filename: "sitemap-pages.xml",
+          entries: [Indexmap::Entry.new(loc: "https://example.com/about")]
+        )
+      ]
+      configuration.index_now.key = "test-key"
+
+      Indexmap::TaskRunner.new(configuration: configuration).create
+
+      assert_equal false, public_path.join("sitemap-pages.xml.gz").exist?
+      assert_includes public_path.join("sitemap.xml").read, "<sitemapindex"
+      assert_equal "test-key\n", public_path.join("test-key.txt").read
+    end
+  end
+
+  def test_write_index_now_key_returns_nil_when_key_is_not_configured
+    Dir.mktmpdir do |dir|
+      configuration = Indexmap::Configuration.new
+      configuration.base_url = "https://example.com"
+      configuration.public_path = Pathname(dir)
+      configuration.sections = [
+        Indexmap::Section.new(
+          filename: "sitemap-pages.xml",
+          entries: [Indexmap::Entry.new(loc: "https://example.com/about")]
+        )
+      ]
+
+      result = Indexmap::TaskRunner.new(configuration: configuration).write_index_now_key
+
+      assert_nil result
+    end
+  end
+end