indexmap 0.8.2 → 0.9.0

data/test/integration/queue_checks.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require "minitest/autorun"
+require "indexmap"
+require "timeout"
+require "rbconfig"
+
+class QueueChecks < Minitest::Test
+  def setup
+    Indexmap::HostConfiguration.apply
+    PublicPage.delete_all
+    5.times { |id| PublicPage.create!(slug: "page-#{id}", published: true) }
+  end
+
+  def test_good_job_executes_database_projection_and_surfaces_validation_failure
+    IntegrationRefreshJob.queue_adapter = :good_job
+    job = IntegrationRefreshJob.perform_later
+    GoodJob.perform_inline
+    assert GoodJob::Job.find(job.provider_job_id).finished_at
+    assert_equal 5, Indexmap::Parser.new.entries.size
+    previous = Indexmap.configuration.storage.read("sitemap.xml")
+    PublicPage.create!(slug: "page-0", published: true)
+    IntegrationRefreshJob.perform_later
+    assert_raises(Indexmap::ValidationError) { GoodJob.perform_inline }
+    assert_equal previous, Indexmap.configuration.storage.read("sitemap.xml")
+  end
+
+  def test_solid_queue_worker_executes_and_records_failed_generation
+    IntegrationRefreshJob.queue_adapter = :solid_queue
+    reader, writer = IO.pipe
+    pid = Process.spawn({"INDEXMAP_ADAPTER" => "solid_queue"}, RbConfig.ruby, "bin/jobs",
+      chdir: Rails.root, out: writer, err: writer, pgroup: true)
+    writer.close
+    job = IntegrationRefreshJob.perform_later
+    wait_for(reader) { SolidQueue::Job.find_by(active_job_id: job.job_id)&.finished_at }
+    assert_equal 5, Indexmap::Parser.new.entries.size
+    previous = Indexmap.configuration.storage.read("sitemap.xml")
+    PublicPage.create!(slug: "page-0", published: true)
+    failed = IntegrationRefreshJob.perform_later
+    wait_for(reader) { SolidQueue::FailedExecution.joins(:job).where(solid_queue_jobs: {active_job_id: failed.job_id}).exists? }
+    assert_equal previous, Indexmap.configuration.storage.read("sitemap.xml")
+  ensure
+    if pid
+      Process.kill("TERM", -pid)
+      Timeout.timeout(15) { Process.wait(pid) }
+    end
+    reader&.close
+    writer&.close unless writer&.closed?
+  end
+
+  def test_active_storage_atomic_publication_and_retention
+    config = Indexmap.configuration
+    config.storage = Indexmap::Storage::ActiveStorage.new(model: SitemapArtifact, public_url: config.base_url)
+    first = Indexmap.create
+    previous = config.storage.read("sitemap.xml")
+    SitemapArtifact.find_by!(filename: "sitemap.xml").file.blob.update!(created_at: 4.days.ago)
+    revision = config.storage.revision("sitemap.xml")
+    config.storage.define_singleton_method(:open) do |filename, &block|
+      raise "Root body must not be downloaded during activation" if filename == "sitemap.xml"
+
+      super(filename, &block)
+    end
+    second = Indexmap.create
+    config.storage.singleton_class.remove_method(:open)
+    refute_equal first, second
+    assert_empty config.storage.prune(root: "sitemap.xml", before: 2.days.ago)
+    assert first.all? { |filename| config.storage.exist?(filename) }
+    assert_raises(Indexmap::SupersededPublication) do
+      Dir.mktmpdir do |directory|
+        staged = Indexmap::Storage::Filesystem.new(path: directory)
+        staged.write("sitemap.xml", previous)
+        config.storage.publish(staged, files: ["sitemap.xml"], root: "sitemap.xml", expected: revision)
+      end
+    end
+    assert Indexmap::Validator.new(configuration: config).validate!
+    config.storage.prune(root: "sitemap.xml", before: Time.now.utc + 1)
+    assert (first - ["sitemap.xml"]).none? { |filename| config.storage.exist?(filename) }
+    assert second.all? { |filename| config.storage.exist?(filename) }
+  end
+
+  def test_active_storage_rejects_stale_builds_without_uploading
+    config = active_storage_configuration("stale.xml")
+    Indexmap.create(configuration: config)
+    expected = config.storage.revision(config.index_filename)
+    Indexmap.create(configuration: config)
+    blobs = ActiveStorage::Blob.order(:id).pluck(:id)
+    assert_raises(Indexmap::SupersededPublication) do
+      config.storage.publish(nil, files: [], root: config.index_filename, expected: expected)
+    end
+    assert_equal blobs, ActiveStorage::Blob.order(:id).pluck(:id)
+  end
+
+  def test_active_storage_cleans_partial_uploads_synchronously
+    config = active_storage_configuration("failed-upload.xml")
+    Indexmap.create(configuration: config)
+    previous = config.storage.read(config.index_filename)
+    blobs = ActiveStorage::Blob.order(:id).pluck(:id)
+    records = SitemapArtifact.order(:id).pluck(:id)
+    service = ActiveStorage::Blob.service
+    uploaded = []
+    service.define_singleton_method(:upload) do |key, io, **options|
+      super(key, io, **options)
+      uploaded << key
+      raise IOError, "upload interrupted after object write" if uploaded.size == 2
+    end
+    assert_raises(IOError) { Indexmap.create(configuration: config) }
+    assert_equal previous, config.storage.read(config.index_filename)
+    assert_equal blobs, ActiveStorage::Blob.order(:id).pluck(:id)
+    assert_equal records, SitemapArtifact.order(:id).pluck(:id)
+    assert uploaded.none? { |key| service.exist?(key) }
+    assert Indexmap::Validator.new(configuration: config).validate!
+  ensure
+    service&.singleton_class&.remove_method(:upload)
+  end
+
+  def test_active_storage_cleans_a_losing_build_without_pruning
+    config = active_storage_configuration("race.xml")
+    storage = config.storage
+    baseline = ActiveStorage::Blob.pluck(:id)
+    lost_files = []
+    lost_keys = []
+    winner = nil
+    storage.define_singleton_method(:publish) do |source, **options|
+      lost_files.concat(options.fetch(:files).reject { |name| name == options.fetch(:root) })
+      source.define_singleton_method(:open) do |filename, &block|
+        if filename == options.fetch(:root)
+          lost_keys.concat(ActiveStorage::Blob.where.not(id: baseline).pluck(:key))
+          competing = Indexmap::Configuration.new
+          competing.base_url = config.base_url
+          competing.index_filename = config.index_filename
+          competing.atomic = true
+          competing.sections = config.sections
+          competing.storage = Indexmap::Storage::ActiveStorage.new(model: SitemapArtifact, public_url: config.base_url)
+          winner = Indexmap.create(configuration: competing)
+        end
+        super(filename, &block)
+      end
+      super(source, **options)
+    end
+    assert_raises(Indexmap::SupersededPublication) { Indexmap.create(configuration: config) }
+    assert_equal winner.size, ActiveStorage::Blob.where.not(id: baseline).count
+    assert lost_files.none? { |filename| storage.exist?(filename) }
+    assert lost_keys.none? { |key| ActiveStorage::Blob.service.exist?(key) }
+    assert winner.all? { |filename| storage.exist?(filename) }
+    assert Indexmap::Validator.new(configuration: config).validate!
+  end
+
+  def test_active_storage_does_not_activate_children_deleted_by_pruning
+    config = active_storage_configuration("prune-race.xml")
+    Indexmap.create(configuration: config)
+    previous = config.storage.read(config.index_filename)
+    blobs = ActiveStorage::Blob.pluck(:id).sort
+    config.storage.define_singleton_method(:publish) do |source, **options|
+      storage = self
+      source.define_singleton_method(:open) do |filename, &block|
+        storage.prune(root: options.fetch(:root), before: Time.now.utc + 1) if filename == options.fetch(:root)
+        super(filename, &block)
+      end
+      super(source, **options)
+    end
+    assert_raises(Indexmap::SupersededPublication) { Indexmap.create(configuration: config) }
+    assert_equal previous, config.storage.read(config.index_filename)
+    assert_equal blobs, ActiveStorage::Blob.pluck(:id).sort
+    assert Indexmap::Validator.new(configuration: config).validate!
+  end
+
+  def test_active_storage_preserves_a_commit_when_a_host_callback_raises
+    config = active_storage_configuration("callback.xml")
+    callback = proc do |record|
+      if record.filename == "callback.xml" && record.file.attached?
+        raise IOError, "host callback failed after commit"
+      end
+    end
+    SitemapArtifact.set_callback(:commit, :after, callback)
+    assert_raises(IOError) { Indexmap.create(configuration: config) }
+    assert config.storage.exist?(config.index_filename)
+    assert Indexmap::Validator.new(configuration: config).validate!
+  ensure
+    SitemapArtifact.skip_callback(:commit, :after, callback) if callback
+  end
+
+  def test_active_storage_reports_cleanup_failures_and_attempts_remaining_objects
+    config = active_storage_configuration("cleanup-error.xml")
+    Indexmap.create(configuration: config)
+    previous = config.storage.read(config.index_filename)
+    service = ActiveStorage::Blob.service
+    uploaded = []
+    attempted = []
+    service.define_singleton_method(:upload) do |key, io, **options|
+      super(key, io, **options)
+      uploaded << key
+      raise IOError, "original upload failure" if uploaded.size == 2
+    end
+    service.define_singleton_method(:delete) do |key|
+      attempted << key
+      raise IOError, "cleanup unavailable" if key == uploaded.first
+
+      super(key)
+    end
+    error = assert_raises(Indexmap::PublicationCleanupError) { Indexmap.create(configuration: config) }
+    assert_equal "original upload failure", error.cause.message
+    assert_equal [uploaded.first], error.failures.map { |failure| failure.fetch(:artifact) }
+    assert_equal uploaded.sort, attempted.sort
+    assert_equal previous, config.storage.read(config.index_filename)
+    assert Indexmap::Validator.new(configuration: config).validate!
+  ensure
+    if service
+      service.singleton_class.remove_method(:upload)
+      service.singleton_class.remove_method(:delete)
+      uploaded&.each { |key| service.delete(key) }
+    end
+  end
+
+  def test_active_storage_publishes_atomic_single_file_outputs
+    config = active_storage_configuration("single.xml")
+    config.format = :single_file
+    config.entries = [{loc: "https://example.com/page"}]
+    assert_equal ["single.xml"], Indexmap.create(configuration: config)
+    assert_equal ["single.xml"], Indexmap.create(configuration: config)
+    assert Indexmap::Validator.new(configuration: config).validate!
+  end
+
+  private
+
+  def active_storage_configuration(root)
+    Indexmap.configuration.tap do |config|
+      config.index_filename = root
+      config.storage = Indexmap::Storage::ActiveStorage.new(model: SitemapArtifact, public_url: config.base_url)
+    end
+  end
+
+  def wait_for(reader)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 30
+    log = +""
+    until yield
+      raise "Worker did not reach expected state: #{log[-2000, 2000]}" if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+
+      if IO.select([reader], nil, nil, 0.1)
+        chunk = reader.read_nonblock(4096, exception: false)
+        log << chunk if chunk.is_a?(String)
+      end
+    end
+  end
+end

data/test/release_task_test.rb

@@ -2,6 +2,8 @@
 
 require "test_helper"
 require "rake"
+require "open3"
+require "rbconfig"
 
 unless respond_to?(:release_version, true)
   load File.expand_path("../Rakefile", __dir__)
@@ -74,8 +76,159 @@ class ReleaseTaskTest < Minitest::Test
     )
   end
 
+  def test_release_preparation_requires_main
+    ["release/0.9.0", "master", ""].each do |branch|
+      error = assert_raises(ArgumentError) { validate_release_branch!(branch) }
+      assert_match(/Prepare releases from main/, error.message)
+    end
+    assert_nil validate_release_branch!("main")
+  end
+
+  def test_prepare_commits_and_pushes_from_main_with_release_hook_exception
+    with_release_repository do |repo, origin|
+      before = git!(repo, "rev-parse", "HEAD").strip
+      history = File.read(File.join(repo, "CHANGELOG.md"))
+      hook = File.join(repo, ".git/hooks/pre-commit")
+      File.write(hook, "#!/bin/sh\nexec ruby script/check_commit_branch.rb\n")
+      FileUtils.chmod(0o755, hook)
+
+      stdout, stderr, status = run_release_task(repo, "release:prepare[minor]")
+      assert status.success?, stdout + stderr
+      expected = release_version("minor")
+      refute_equal before, git!(repo, "rev-parse", "HEAD").strip
+      assert_equal git!(repo, "rev-parse", "HEAD"), git!(origin, "rev-parse", "main")
+      assert_equal "CHANGELOG.md\nlib/indexmap/version.rb\n",
+        git!(repo, "diff-tree", "--no-commit-id", "--name-only", "-r", "HEAD")
+      assert_includes File.read(File.join(repo, "lib/indexmap/version.rb")), expected
+      assert_includes File.read(File.join(repo, "CHANGELOG.md")), "## [#{expected}]"
+      assert File.read(File.join(repo, "CHANGELOG.md")).end_with?(history[history.index("## [")..])
+      message = git!(repo, "log", "-1", "--format=%B")
+      assert_includes message, "chore(release): prepare v#{expected}"
+      %w[What: Why: Where:].each { |heading| assert_includes message, heading }
+      assert_empty git!(repo, "status", "--porcelain")
+      assert_empty git!(repo, "tag", "--list")
+      assert_empty git!(origin, "tag", "--list")
+      assert_includes stdout, "release:tag"
+
+      stdout, stderr, status = run_release_task(repo, "release:tag")
+      assert status.success?, stdout + stderr
+      assert_equal git!(repo, "rev-parse", "HEAD"), git!(origin, "rev-parse", "v#{expected}^{}")
+    end
+  end
+
+  def test_prepare_keeps_commit_hooks_enabled_and_does_not_push_when_they_fail
+    with_release_repository do |repo, origin|
+      before = git!(origin, "rev-parse", "main")
+      hook = File.join(repo, ".git/hooks/commit-msg")
+      File.write(hook, "#!/bin/sh\necho 'commit hook rejected release' >&2\nexit 1\n")
+      FileUtils.chmod(0o755, hook)
+
+      stdout, stderr, status = run_release_task(repo, "release:prepare[minor]")
+      refute status.success?, stdout + stderr
+      assert_includes stderr, "commit hook rejected release"
+      assert_equal before, git!(repo, "rev-parse", "HEAD")
+      assert_equal before, git!(origin, "rev-parse", "main")
+      assert_empty git!(origin, "tag", "--list")
+    end
+  end
+
+  def test_branch_hook_rejects_ordinary_main_commits_even_with_release_flag
+    with_release_repository do |repo, _origin|
+      File.write(File.join(repo, "unrelated.txt"), "unrelated")
+      git!(repo, "add", "unrelated.txt")
+      [nil, "1"].each do |flag|
+        _stdout, stderr, status = Open3.capture3(
+          {"INDEXMAP_RELEASE_COMMIT" => flag},
+          RbConfig.ruby, "script/check_commit_branch.rb", chdir: repo
+        )
+        refute status.success?
+        assert_includes stderr, "Direct commits to 'main' are not allowed"
+      end
+    end
+  end
+
+  def test_tag_pushes_only_the_prepared_merged_commit
+    with_release_repository do |repo, origin|
+      before = git!(repo, "rev-parse", "HEAD").strip
+      stdout, stderr, status = run_release_task(repo, "release:tag")
+      assert status.success?, stdout + stderr
+      assert_equal before, git!(repo, "rev-parse", "HEAD").strip
+      assert_equal before, git!(repo, "rev-parse", "v#{Indexmap::VERSION}^{}").strip
+      assert_includes git!(origin, "tag", "--list"), "v#{Indexmap::VERSION}"
+      assert_empty git!(repo, "status", "--porcelain")
+    end
+  end
+
+  def test_prepare_rejects_topic_and_detached_branches_without_changing_files
+    with_release_repository do |repo, origin|
+      before = git!(repo, "rev-parse", "HEAD").strip
+      git!(repo, "checkout", "-b", "release/next")
+      ["release/next", before].each do |ref|
+        git!(repo, "checkout", ref)
+        stdout, stderr, status = run_release_task(repo, "release:prepare[minor]")
+        refute status.success?, stdout + stderr
+        assert_includes stderr, "Prepare releases from main"
+        assert_equal before, git!(repo, "rev-parse", "HEAD").strip
+        assert_empty git!(repo, "status", "--porcelain")
+        assert_empty git!(origin, "tag", "--list")
+      end
+    end
+  end
+
+  def test_tag_rejects_main_that_has_not_been_merged_to_origin
+    with_release_repository do |repo, origin|
+      File.write(File.join(repo, "unmerged.txt"), "unmerged")
+      git!(repo, "add", "unmerged.txt")
+      git!(repo, "commit", "-m", "test: unmerged change")
+      %w[release:prepare[minor] release:tag].each do |task|
+        stdout, stderr, status = run_release_task(repo, task)
+        refute status.success?, stdout + stderr
+        assert_includes stderr, "Local main must match origin/main"
+        assert_empty git!(repo, "status", "--porcelain")
+      end
+      assert_empty git!(repo, "tag", "--list")
+      assert_empty git!(origin, "tag", "--list")
+    end
+  end
+
   private
 
+  def with_release_repository
+    Dir.mktmpdir("indexmap-release-test") do |directory|
+      repo = File.join(directory, "repo")
+      origin = File.join(directory, "origin.git")
+      FileUtils.mkdir_p([repo, origin, File.join(repo, "lib/indexmap")])
+      git!(origin, "init", "--bare")
+      git!(repo, "init", "-b", "main")
+      git!(repo, "config", "user.name", "Release Test")
+      git!(repo, "config", "user.email", "release@example.test")
+      %w[Rakefile indexmap.gemspec CHANGELOG.md cliff.toml lib/indexmap/version.rb script/check_commit_branch.rb].each do |file|
+        FileUtils.mkdir_p(File.dirname(File.join(repo, file)))
+        FileUtils.cp(File.expand_path("../#{file}", __dir__), File.join(repo, file))
+      end
+      git!(repo, "add", ".")
+      git!(repo, "commit", "-m", "test: prepare release fixture")
+      git!(repo, "remote", "add", "origin", origin)
+      git!(repo, "push", "-u", "origin", "main")
+      yield repo, origin
+    end
+  end
+
+  def git!(directory, *args)
+    stdout, stderr, status = Open3.capture3("git", *args, chdir: directory)
+    raise stderr unless status.success?
+
+    stdout
+  end
+
+  def run_release_task(directory, task)
+    Open3.capture3(
+      {"BUNDLE_GEMFILE" => File.expand_path(ENV.fetch("BUNDLE_GEMFILE", "Gemfile"))},
+      RbConfig.ruby, "-rrake", "-e", 'load "Rakefile"; Rake.application.invoke_task(ARGV.fetch(0))', task,
+      chdir: directory
+    )
+  end
+
   def local_release_tag_exists?(_tag)
     @local_release_tag_exists || false
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexmap
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.9.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -175,9 +175,17 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- UPGRADE.md
+- docs/architecture.md
+- docs/deployment.md
+- docs/domain-terms.md
+- docs/dynamic-sitemaps.md
+- docs/search-engine-ping.md
+- docs/storage.md
 - lib/generators/indexmap/install_generator.rb
 - lib/generators/indexmap/templates/initializer.rb.tt
 - lib/indexmap.rb
+- lib/indexmap/artifact.rb
 - lib/indexmap/configuration.rb
 - lib/indexmap/creator.rb
 - lib/indexmap/entry.rb
@@ -188,28 +196,37 @@ files:
 - lib/indexmap/pinger/base.rb
 - lib/indexmap/pinger/google.rb
 - lib/indexmap/pinger/index_now.rb
+- lib/indexmap/publication.rb
 - lib/indexmap/railtie.rb
+- lib/indexmap/robots.rb
+- lib/indexmap/robots_controller.rb
 - lib/indexmap/section.rb
+- lib/indexmap/sitemaps_controller.rb
 - lib/indexmap/storage/active_storage.rb
-- lib/indexmap/storage/file.rb
 - lib/indexmap/storage/filesystem.rb
 - lib/indexmap/storage/memory.rb
 - lib/indexmap/task_runner.rb
 - lib/indexmap/tasks.rb
 - lib/indexmap/validator.rb
+- lib/indexmap/validator/duplicates.rb
 - lib/indexmap/version.rb
 - lib/indexmap/writer.rb
 - lib/tasks/indexmap_tasks.rake
 - test/indexmap/configuration_test.rb
+- test/indexmap/correctness_test.rb
 - test/indexmap/install_generator_test.rb
 - test/indexmap/lazy_loading_test.rb
 - test/indexmap/parser_test.rb
 - test/indexmap/pinger/google_test.rb
 - test/indexmap/pinger/index_now_test.rb
+- test/indexmap/publication_test.rb
+- test/indexmap/robots_test.rb
+- test/indexmap/sitemaps_controller_test.rb
 - test/indexmap/storage_test.rb
 - test/indexmap/task_runner_test.rb
 - test/indexmap/validator_test.rb
 - test/indexmap/writer_test.rb
+- test/integration/queue_checks.rb
 - test/release_task_test.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/indexmap
@@ -219,8 +236,8 @@ metadata:
   homepage_uri: https://www.ethos-link.com/opensource/indexmap
   source_code_uri: https://github.com/ethos-link/indexmap
   bug_tracker_uri: https://github.com/ethos-link/indexmap/issues
-  changelog_uri: https://github.com/ethos-link/indexmap/blob/main/CHANGELOG.md
-  documentation_uri: https://github.com/ethos-link/indexmap/blob/main/README.md
+  changelog_uri: https://github.com/ethos-link/indexmap/blob/v0.9.0/CHANGELOG.md
+  documentation_uri: https://github.com/ethos-link/indexmap/blob/v0.9.0/README.md
   funding_uri: https://www.reviato.com/
   github_repo: ssh://github.com/ethos-link/indexmap
   allowed_push_host: https://rubygems.org
@@ -239,7 +256,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Generate sitemap indexes and child sitemaps with plain Ruby
 test_files: []

data/lib/indexmap/storage/file.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Indexmap
-  module Storage
-    File = Struct.new(:filename, :body, :content_type) do
-      def basename
-        ::File.basename(filename)
-      end
-    end
-  end
-end