indexmap 0.8.2 → 0.9.0

data/test/indexmap/publication_test.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "timeout"
+
+class IndexmapPublicationTest < Minitest::Test
+  def configuration(directory, atomic: true)
+    Indexmap::Configuration.new.tap do |config|
+      config.base_url = "https://example.com"
+      config.storage = Indexmap::Storage::Filesystem.new(path: directory, public_url: config.base_url)
+      config.atomic = atomic
+      config.max_urls = 2
+      config.sections = -> do
+        entries = (1..5).lazy.map { |id| Indexmap::Entry.new(loc: "https://example.com/#{id}") }
+        [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: entries)]
+      end
+    end
+  end
+
+  def test_partitions_once_and_publishes_immutable_children
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      result = Indexmap::TaskRunner.new(configuration: config).create
+      assert_equal [2, 2, 1, 3], result.fetch(:sitemaps).map { |file| file.fetch(:link_count) }
+      old_files = result.fetch(:files)
+      bodies = old_files.to_h { |file| [file, config.storage.read(file)] }
+      assert_raises(Indexmap::ConfigurationError) { config.storage.write(old_files.first, "overwrite") }
+      assert_raises(Indexmap::ConfigurationError) { Indexmap::TaskRunner.new(configuration: config).format }
+      second = Indexmap::TaskRunner.new(configuration: config).create
+      refute_equal old_files, second.fetch(:files)
+      old_files.reject { |file| file == "sitemap.xml" }.each { |file| assert_equal bodies[file], config.storage.read(file) }
+      publication = Indexmap::Publication.new(root: "sitemap.xml")
+      assert config.storage.exist?(publication.archive(publication.generation(old_files.first)))
+      assert Indexmap::Validator.new(configuration: config).validate!
+    end
+  end
+
+  def test_byte_partitioning_counts_escaped_output
+    Dir.mktmpdir do |directory|
+      config = configuration(directory, atomic: false)
+      config.max_urls = 50_000
+      config.max_bytes = 400
+      config.sections = [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: (1..3).map do |id|
+        Indexmap::Entry.new(loc: "https://example.com/#{id}#{"a" * 100}&b")
+      end)]
+      files = Indexmap.create(configuration: config)
+      assert_operator files.size, :>, 2
+      files.each { |file| assert_operator config.storage.metadata(file).fetch(:bytes), :<=, 400 }
+    end
+  end
+
+  def test_failed_copy_does_not_change_published_files_or_leave_staging
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      Indexmap.create(configuration: config)
+      storage = config.storage
+      previous = storage.list.to_h { |filename| [filename, storage.read(filename)] }
+      storage.define_singleton_method(:publish) do |source, **options|
+        calls = 0
+        source.define_singleton_method(:open) do |filename, &block|
+          calls += 1
+          raise IOError, "interrupted copy" if calls == 2
+
+          super(filename, &block)
+        end
+        super(source, **options)
+      end
+      assert_raises(IOError) { Indexmap.create(configuration: config) }
+      assert_equal previous, storage.list.to_h { |filename| [filename, storage.read(filename)] }
+      assert_empty Dir.glob(File.join(directory, ".indexmap-builds/*/build-*"))
+    end
+  end
+
+  def test_atomic_refresh_preserves_existing_root_permissions
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      Indexmap.create(configuration: config)
+      root = File.join(directory, "sitemap.xml")
+      File.chmod(0o640, root)
+      Indexmap.create(configuration: config)
+      assert_equal 0o640, File.stat(root).mode & 0o777
+      assert Indexmap::Validator.new(configuration: config).validate!
+    end
+  end
+
+  def test_failed_root_activation_removes_new_children
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      old = Indexmap.create(configuration: config)
+      previous = config.storage.read("sitemap.xml")
+      rename = File.method(:rename)
+      File.stub(:rename, ->(source, destination) do
+        raise Errno::EACCES, "root activation denied" if destination.to_s == File.join(directory, "sitemap.xml")
+
+        rename.call(source, destination)
+      end) do
+        assert_raises(Errno::EACCES) { Indexmap.create(configuration: config) }
+      end
+      assert_equal previous, config.storage.read("sitemap.xml")
+      children = config.storage.list(prefix: "sitemap-", suffix: ".xml").reject { |name| name.end_with?("-index.xml") }
+      assert_equal (old - ["sitemap.xml"]).sort, children
+      assert_empty Dir.glob(File.join(directory, ".indexmap-builds/*/build-*"))
+      assert Indexmap::Validator.new(configuration: config).validate!
+    end
+  end
+
+  def test_copying_does_not_block_a_competing_publisher
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      Indexmap.create(configuration: config)
+      storage = config.storage
+      entered = Queue.new
+      release = Queue.new
+      storage.define_singleton_method(:publish) do |source, **options|
+        source.define_singleton_method(:open) do |filename, &block|
+          unless @paused
+            @paused = true
+            entered << true
+            release.pop
+          end
+          super(filename, &block)
+        end
+        super(source, **options)
+      end
+      thread = Thread.new do
+        Indexmap.create(configuration: config)
+      rescue Indexmap::SupersededPublication => error
+        error
+      end
+      Timeout.timeout(5) { entered.pop }
+      competing = configuration(directory)
+      winner = Timeout.timeout(5) { Indexmap.create(configuration: competing) }
+      release << true
+      assert_instance_of Indexmap::SupersededPublication, Timeout.timeout(5) { thread.value }
+      assert winner.all? { |filename| storage.exist?(filename) }
+      assert Indexmap::Parser.new(storage: storage).entries.all? { |entry| winner.include?(entry.source_sitemap) }
+      assert Indexmap::Validator.new(configuration: config).validate!
+      assert_empty Dir.glob(File.join(directory, ".indexmap-builds/*/build-*"))
+    ensure
+      thread&.kill if thread&.alive?
+    end
+  end
+
+  def test_pruning_reclaims_crashed_staging_and_skips_live_builds
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      Indexmap.create(configuration: config)
+      reader, writer = IO.pipe
+      pid = fork do
+        reader.close
+        config.storage.define_singleton_method(:publish) do |source, **options|
+          source.define_singleton_method(:open) do |filename, &block|
+            writer.write("staging")
+            writer.flush
+            sleep
+            super(filename, &block)
+          end
+          super(source, **options)
+        end
+        Indexmap.create(configuration: config)
+      end
+      writer.close
+      assert IO.select([reader], nil, nil, 5), "build did not reach destination staging"
+      reader.read(7)
+      staged = Dir.glob(File.join(directory, ".indexmap-builds/*/build-*")).fetch(0)
+      File.utime(Time.at(0), Time.at(0), staged)
+      assert_empty config.storage.prune(root: "sitemap.xml", before: Time.now.utc - 3600)
+      assert File.directory?(staged)
+      Process.kill("KILL", pid)
+      Process.wait(pid)
+      pid = nil
+      deleted = config.storage.prune(root: "sitemap.xml", before: Time.now.utc - 3600)
+      assert_equal [Pathname(staged).relative_path_from(Pathname(directory)).to_s], deleted
+      refute File.exist?(staged)
+      assert Indexmap::Validator.new(configuration: config).validate!
+    ensure
+      if pid
+        Process.kill("KILL", pid)
+        Process.wait(pid)
+      end
+      reader&.close
+      writer&.close unless writer&.closed?
+    end
+  end
+
+  def test_rejects_stale_builds_and_keeps_other_outputs_during_cleanup
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      old = Indexmap.create(configuration: config)
+      config.output(:other) do |output|
+        output.index_filename = "other.xml"
+        output.sections = -> { [Indexmap::Section.new(filename: "sitemap-other.xml", entries: [{loc: "https://example.com/other"}])] }
+      end
+      other = Indexmap.create(:other, configuration: config)
+      expected = config.storage.revision("sitemap.xml")
+      Indexmap.create(configuration: config)
+      assert_raises(Indexmap::SupersededPublication) do
+        config.storage.publish(nil, files: [], root: "sitemap.xml", expected: expected)
+      end
+      config.storage.list(suffix: ".xml").each { |name| File.utime(Time.at(0), Time.at(0), File.join(directory, name)) }
+      deleted = config.storage.prune(root: "sitemap.xml", before: Time.now.utc - 3600)
+      assert (old - ["sitemap.xml"]).all? { |file| deleted.include?(file) }
+      assert other.all? { |file| config.storage.exist?(file) }
+    end
+  end
+
+  def test_duplicates_across_partitions_fail_before_publication
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      Indexmap.create(configuration: config)
+      previous = config.storage.read("sitemap.xml")
+      config.sections = [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: [
+        {loc: "https://example.com/a"}, {loc: "https://example.com/b"}, {loc: "https://example.com/a"}
+      ])]
+      assert_raises(Indexmap::ValidationError) { Indexmap.create(configuration: config) }
+      assert_equal previous, config.storage.read("sitemap.xml")
+    end
+  end
+
+  def test_filesystem_rejects_symlinks_escaping_storage
+    Dir.mktmpdir do |directory|
+      Dir.mktmpdir do |outside|
+        storage = Indexmap::Storage::Filesystem.new(path: directory)
+        File.symlink(outside, File.join(directory, "escape"))
+        assert_raises(ArgumentError) { storage.write("escape/private.xml", "private") }
+      end
+    end
+  end
+
+  def test_concurrent_builds_cannot_overwrite_a_newer_root
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      entered = Queue.new
+      release = Queue.new
+      config.sections = -> do
+        source = Enumerator.new do |stream|
+          entered << true
+          release.pop
+          stream << Indexmap::Entry.new(loc: "https://example.com/page")
+        end
+        [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: source)]
+      end
+      threads = 2.times.map do
+        Thread.new do
+          Indexmap.create(configuration: config)
+        rescue Indexmap::SupersededPublication => error
+          error
+        end
+      end
+      Timeout.timeout(5) { 2.times { entered.pop } }
+      2.times { release << true }
+      results = Timeout.timeout(5) { threads.map(&:value) }
+      assert_equal 1, results.count { |result| result.is_a?(Indexmap::SupersededPublication) }
+      assert Indexmap::Validator.new(configuration: config).validate!
+    ensure
+      threads&.each { |thread| thread.kill if thread.alive? }
+    end
+  end
+
+  def test_atomic_mode_requires_an_explicit_adapter_capability
+    config = Indexmap::Configuration.new
+    config.atomic = true
+    config.storage = Indexmap::Storage::Memory.new
+    config.sections = -> { raise "must not evaluate an unsupported build" }
+    assert_raises(Indexmap::ConfigurationError) { Indexmap.create(configuration: config) }
+  end
+
+  def test_duplicates_across_sort_runs_fail_before_publication
+    Dir.mktmpdir do |directory|
+      config = configuration(directory)
+      Indexmap.create(configuration: config)
+      previous = config.storage.read("sitemap.xml")
+      config.max_urls = 5000
+      entries = (0..10_000).lazy.map { |id| {loc: "https://example.com/#{id % 10_000}"} }
+      config.sections = [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: entries)]
+      assert_raises(Indexmap::ValidationError) { Indexmap.create(configuration: config) }
+      assert_equal previous, config.storage.read("sitemap.xml")
+    end
+  end
+end

data/test/indexmap/robots_test.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "indexmap/robots_controller"
+require "rack/mock"
+
+class IndexmapRobotsTest < Minitest::Test
+  def setup
+    Indexmap.reset!
+    Indexmap.configuration.base_url = "https://example.com"
+  end
+
+  def teardown
+    Indexmap.reset!
+  end
+
+  def test_defaults_to_disallow_without_resolving_sitemap_data
+    Indexmap.configuration.base_url = -> { raise "must not resolve" }
+    Indexmap.configuration.sections = -> { raise "must not query" }
+    Indexmap.configuration.storage = -> { raise "must not read storage" }
+    assert_equal "User-agent: *\nDisallow: /\n", Indexmap.configuration.robots.body
+  end
+
+  def test_uses_explicit_indexability_and_the_configured_root
+    indexable = true
+    config = Indexmap.configuration
+    config.index_filename = "sitemaps/main.xml"
+    config.robots.indexable = -> { indexable }
+    config.robots.groups = [{user_agent: ["*", "ExampleBot"], allow: ["/"], disallow: ["/admin/", "/search"]}]
+    assert_includes config.robots.body, "Sitemap: https://example.com/sitemaps/main.xml"
+    assert_includes config.robots.body, "Disallow: /admin/"
+    assert_includes config.robots.body, "User-agent: ExampleBot"
+    indexable = false
+    assert_equal "User-agent: *\nDisallow: /\n", config.robots.body
+  end
+
+  def test_rejects_injected_directives_and_invalid_sitemap_urls
+    config = Indexmap.configuration
+    config.robots.indexable = true
+    config.robots.groups = [{user_agent: "*", disallow: ["/admin/\nAllow: /"]}]
+    assert_raises(Indexmap::ConfigurationError) { config.robots.body }
+    config.robots.groups = [{user_agent: "*"}]
+    ["/sitemap.xml", "https://user:password@example.com/sitemap.xml", "https://example.com/sitemap.xml\nDisallow: /", "file:///sitemap.xml"].each do |url|
+      config.robots.sitemaps = [url]
+      assert_raises(Indexmap::ConfigurationError) { config.robots.body }
+    end
+  end
+
+  def test_opt_in_route_serves_get_head_and_conditional_responses_without_cookies
+    Indexmap.configuration.robots.indexable = true
+    Indexmap.configuration.sections = -> { raise "must not query content" }
+    routes = ActionDispatch::Routing::RouteSet.new
+    routes.draw { get "/robots.txt", to: Indexmap::RobotsController.action(:show) }
+    client = Rack::MockRequest.new(Rack::Head.new(routes))
+    response = client.get("/robots.txt", "HTTP_HOST" => "attacker.example")
+    assert_equal 200, response.status
+    assert_match "text/plain", response["content-type"]
+    assert_includes response.body, "Sitemap: https://example.com/sitemap.xml"
+    assert_includes response["cache-control"], "max-age=3600"
+    assert_nil response["set-cookie"]
+    head = client.head("/robots.txt")
+    assert_equal 200, head.status
+    assert_empty head.body
+    cached = client.get("/robots.txt", "HTTP_IF_NONE_MATCH" => response["etag"])
+    assert_equal 304, cached.status
+    assert_empty cached.body
+    assert_equal 404, client.post("/robots.txt").status
+    Indexmap.configuration.robots.indexable = false
+    disabled = client.get("/robots.txt", "HTTP_IF_NONE_MATCH" => response["etag"])
+    assert_equal 200, disabled.status
+    assert_equal "User-agent: *\nDisallow: /\n", disabled.body
+  end
+end

data/test/indexmap/sitemaps_controller_test.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "indexmap/sitemaps_controller"
+require "rack/mock"
+require "active_support/cache"
+
+class IndexmapSitemapsControllerTest < Minitest::Test
+  def test_serves_only_published_and_retained_artifacts_without_source_queries
+    Dir.mktmpdir do |directory|
+      Indexmap.reset!
+      Rails.cache = ActiveSupport::Cache::MemoryStore.new
+      config = Indexmap.configuration
+      config.base_url = "https://example.com"
+      config.storage = Indexmap::Storage::Filesystem.new(path: directory)
+      config.atomic = true
+      config.sections = -> { [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: [{loc: "https://example.com/page"}])] }
+      old = Indexmap.create.first
+      child = Indexmap.create.first
+      unpublished = Indexmap::Publication.new(root: "sitemap.xml").filename(generation: "f" * 32, section: 1, part: 1)
+      config.storage.write(unpublished, "not published")
+      config.sections = -> { raise "must not query source content" }
+      routes = ActionDispatch::Routing::RouteSet.new
+      routes.draw do
+        get "/sitemap.xml", to: Indexmap::SitemapsController.action(:show), defaults: {filename: "sitemap.xml"}, format: false
+        get "/:filename", to: Indexmap::SitemapsController.action(:show), constraints: {filename: /[^\/]+/}, format: false
+      end
+      client = Rack::MockRequest.new(Rack::Head.new(routes))
+      response = client.get("/#{child}")
+      assert_equal 200, response.status
+      assert_includes response.body, "https://example.com/page"
+      assert_match "application/xml", response["content-type"]
+      assert_includes response["cache-control"], "immutable"
+      assert_nil response["set-cookie"]
+      assert_equal 200, client.get("/#{old}").status
+      assert_equal 404, client.get("/#{unpublished}").status
+      assert_equal 404, client.get("/robots.txt").status
+      assert_equal 200, client.get("/#{child}?prefix=../../").status
+      config.storage.define_singleton_method(:open) { |_filename, &_block| raise "must not download body" }
+      cached = client.get("/#{child}", "HTTP_IF_NONE_MATCH" => response["etag"])
+      assert_equal 304, cached.status
+      assert_empty cached.body
+      head = client.head("/#{child}")
+      assert_equal 200, head.status
+      assert_empty head.body
+    ensure
+      Indexmap.reset!
+    end
+  end
+end

data/test/indexmap/storage_test.rb

@@ -94,6 +94,33 @@ class IndexmapStorageTest < Minitest::Test
     end
   end
 
+  def test_filesystem_listing_scopes_literal_prefixes_before_inspecting_files
+    Dir.mktmpdir do |directory|
+      storage = Indexmap::Storage::Filesystem.new(path: directory)
+      storage.write("sitemaps/current-1.xml", "xml")
+      storage.write("sitemaps/current-2.xml", "xml")
+      storage.write("sitemaps/other/old.xml", "xml")
+      storage.write("uploads/unrelated.xml", "xml")
+      storage.write("sitemaps/literal[1].xml", "xml")
+      file_test = FileTest.method(:file?)
+      FileTest.stub(:file?, ->(name) do
+        raise "inspected unrelated files" if name.to_s.include?("uploads/") || name.to_s.include?("sitemaps/other/")
+
+        file_test.call(name)
+      end) do
+        assert_equal ["sitemaps/current-1.xml", "sitemaps/current-2.xml"], storage.list(prefix: "sitemaps/current-", suffix: ".xml")
+      end
+      assert_equal ["sitemaps/literal[1].xml"], storage.list(prefix: "sitemaps/literal[", suffix: ".xml")
+      assert_raises(ArgumentError) { storage.list(prefix: "../outside/") }
+      assert_raises(ArgumentError) { storage.list(prefix: "/sitemaps/") }
+      assert_raises(ArgumentError) { storage.list(prefix: "sitemaps/../uploads/") }
+      storage.write(".private/hidden.xml", "private")
+      File.symlink(File.join(directory, "sitemaps"), File.join(directory, "alias"))
+      assert_empty storage.list(prefix: ".private/")
+      assert_empty storage.list(prefix: "alias/")
+    end
+  end
+
   def test_filesystem_storage_rejects_absolute_filenames
     Dir.mktmpdir do |dir|
       storage = Indexmap::Storage::Filesystem.new(path: dir)

data/test/indexmap/task_runner_test.rb

@@ -116,6 +116,30 @@ class IndexmapTaskRunnerTest < Minitest::Test
     assert_includes storage.read("sitemap.xml"), "\n  <url>"
   end
 
+  def test_generation_reports_do_not_download_published_artifacts
+    storage = Class.new(Indexmap::Storage::Memory) do
+      def read(_filename)
+        raise "must not download published artifacts for counts"
+      end
+    end.new
+    result = Indexmap::TaskRunner.new(configuration: configuration_with(storage: storage)).create
+    assert_equal [1, 1], result.fetch(:sitemaps).map { |file| file.fetch(:link_count) }
+  end
+
+  def test_create_resolves_storage_once_for_publication_and_key_handling
+    stores = []
+    config = configuration_with
+    config.storage = -> { stores.push(Indexmap::Storage::Memory.new).last }
+    config.index_now.key = VALID_KEY
+    result = Indexmap::TaskRunner.new(configuration: config).create
+    assert_equal 1, stores.size
+    assert result.fetch(:files).all? { |filename| stores.first.exist?(filename) }
+    assert_equal VALID_KEY, stores.first.read("#{VALID_KEY}.txt")
+    Indexmap::TaskRunner.new(configuration: config).create
+    assert_equal 2, stores.size
+    assert stores.last.exist?("sitemap.xml")
+  end
+
   private
 
   def configuration_with(storage: Indexmap::Storage::Memory.new)

data/test/indexmap/validator_test.rb

@@ -167,6 +167,20 @@ class IndexmapValidatorTest < Minitest::Test
     XML
   end
 
+  def test_validation_keeps_one_resolved_storage_for_the_entire_graph
+    config = configuration_with
+    config.sections = [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: [{loc: "https://example.com/"}])]
+    Indexmap.create(configuration: config)
+    storage = config.storage
+    calls = 0
+    config.storage = -> {
+      calls += 1
+      storage
+    }
+    assert Indexmap::Validator.new(configuration: config).validate!(collect: false)
+    assert_equal 1, calls
+  end
+
   private
 
   def validate_sitemap(body)

data/test/indexmap/writer_test.rb

@@ -85,4 +85,46 @@ class IndexmapWriterTest < Minitest::Test
 
     refute_includes index_xml, "<lastmod>"
   end
+
+  def test_consumes_section_entries_once_and_preserves_output_and_latest_date
+    entries = [
+      {loc: "https://example.com/first", lastmod: Date.new(2026, 4, 21)},
+      {loc: "https://example.com/second", lastmod: "2026-04-22T09:15:00Z"},
+      {loc: "https://example.com/third", lastmod: nil}
+    ]
+    iterations = 0
+    source = Enumerator.new do |yielder|
+      iterations += 1
+      raise "source was enumerated twice" if iterations > 1
+
+      entries.each { |entry| yielder << entry }
+    end
+    expected = Indexmap::Writer.new(base_url: "https://example.com",
+      sections: [{filename: "sitemap-pages.xml", entries: entries}]).write
+    actual = Indexmap::Writer.new(base_url: "https://example.com",
+      sections: [{filename: "sitemap-pages.xml", entries: source}]).write
+
+    assert_equal expected.map(&:body), actual.map(&:body)
+    assert_equal 1, iterations
+    assert_includes actual.last.body, "<lastmod>2026-04-22T09:15:00Z</lastmod>"
+  end
+
+  def test_create_publishes_a_one_shot_section_source
+    configuration = Indexmap::Configuration.new
+    configuration.base_url = "https://example.com"
+    configuration.storage = Indexmap::Storage::Memory.new
+    iterations = 0
+    source = Enumerator.new do |yielder|
+      iterations += 1
+      raise "source was enumerated twice" if iterations > 1
+
+      yielder << Indexmap::Entry.new(loc: "https://example.com/page", lastmod: Time.utc(2026, 4, 21))
+    end
+    configuration.sections = [Indexmap::Section.new(filename: "sitemap-pages.xml", entries: source)]
+
+    assert_equal ["sitemap-pages.xml", "sitemap.xml"], Indexmap.create(configuration: configuration)
+    assert_equal 1, iterations
+    assert_includes configuration.storage.read("sitemap.xml"), "<lastmod>2026-04-21T00:00:00Z</lastmod>"
+    assert_includes configuration.storage.read("sitemap-pages.xml"), "<loc>https://example.com/page</loc>"
+  end
 end