indexmap 0.8.2 → 0.10.0

data/test/integration/queue_checks.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require "minitest/autorun"
+require "indexmap"
+require "timeout"
+require "rbconfig"
+
+class QueueChecks < Minitest::Test
+  def setup
+    Indexmap::HostConfiguration.apply
+    PublicPage.delete_all
+    5.times { |id| PublicPage.create!(slug: "page-#{id}", published: true) }
+  end
+
+  def test_good_job_executes_database_projection_and_surfaces_validation_failure
+    IntegrationRefreshJob.queue_adapter = :good_job
+    job = IntegrationRefreshJob.perform_later
+    GoodJob.perform_inline
+    assert GoodJob::Job.find(job.provider_job_id).finished_at
+    assert_equal 5, Indexmap::Parser.new.entries.size
+    previous = Indexmap.configuration.storage.read("sitemap.xml")
+    PublicPage.create!(slug: "page-0", published: true)
+    IntegrationRefreshJob.perform_later
+    assert_raises(Indexmap::ValidationError) { GoodJob.perform_inline }
+    assert_equal previous, Indexmap.configuration.storage.read("sitemap.xml")
+  end
+
+  def test_solid_queue_worker_executes_and_records_failed_generation
+    IntegrationRefreshJob.queue_adapter = :solid_queue
+    reader, writer = IO.pipe
+    pid = Process.spawn({"INDEXMAP_ADAPTER" => "solid_queue"}, RbConfig.ruby, "bin/jobs",
+      chdir: Rails.root, out: writer, err: writer, pgroup: true)
+    writer.close
+    job = IntegrationRefreshJob.perform_later
+    wait_for(reader) { SolidQueue::Job.find_by(active_job_id: job.job_id)&.finished_at }
+    assert_equal 5, Indexmap::Parser.new.entries.size
+    previous = Indexmap.configuration.storage.read("sitemap.xml")
+    PublicPage.create!(slug: "page-0", published: true)
+    failed = IntegrationRefreshJob.perform_later
+    wait_for(reader) { SolidQueue::FailedExecution.joins(:job).where(solid_queue_jobs: {active_job_id: failed.job_id}).exists? }
+    assert_equal previous, Indexmap.configuration.storage.read("sitemap.xml")
+  ensure
+    if pid
+      Process.kill("TERM", -pid)
+      Timeout.timeout(15) { Process.wait(pid) }
+    end
+    reader&.close
+    writer&.close unless writer&.closed?
+  end
+
+  def test_active_storage_atomic_publication_and_retention
+    config = Indexmap.configuration
+    config.storage = Indexmap::Storage::ActiveStorage.new(model: SitemapArtifact, public_url: config.base_url)
+    first = Indexmap.create
+    previous = config.storage.read("sitemap.xml")
+    SitemapArtifact.find_by!(filename: "sitemap.xml").file.blob.update!(created_at: 4.days.ago)
+    revision = config.storage.revision("sitemap.xml")
+    config.storage.define_singleton_method(:open) do |filename, &block|
+      raise "Root body must not be downloaded during activation" if filename == "sitemap.xml"
+
+      super(filename, &block)
+    end
+    second = Indexmap.create
+    config.storage.singleton_class.remove_method(:open)
+    refute_equal first, second
+    assert_empty config.storage.prune(root: "sitemap.xml", before: 2.days.ago)
+    assert first.all? { |filename| config.storage.exist?(filename) }
+    assert_raises(Indexmap::SupersededPublication) do
+      Dir.mktmpdir do |directory|
+        staged = Indexmap::Storage::Filesystem.new(path: directory)
+        staged.write("sitemap.xml", previous)
+        config.storage.publish(staged, files: ["sitemap.xml"], root: "sitemap.xml", expected: revision)
+      end
+    end
+    assert Indexmap::Validator.new(configuration: config).validate!
+    config.storage.prune(root: "sitemap.xml", before: Time.now.utc + 1)
+    assert (first - ["sitemap.xml"]).none? { |filename| config.storage.exist?(filename) }
+    assert second.all? { |filename| config.storage.exist?(filename) }
+  end
+
+  def test_active_storage_rejects_stale_builds_without_uploading
+    config = active_storage_configuration("stale.xml")
+    Indexmap.create(configuration: config)
+    expected = config.storage.revision(config.index_filename)
+    Indexmap.create(configuration: config)
+    blobs = ActiveStorage::Blob.order(:id).pluck(:id)
+    assert_raises(Indexmap::SupersededPublication) do
+      config.storage.publish(nil, files: [], root: config.index_filename, expected: expected)
+    end
+    assert_equal blobs, ActiveStorage::Blob.order(:id).pluck(:id)
+  end
+
+  def test_active_storage_cleans_partial_uploads_synchronously
+    config = active_storage_configuration("failed-upload.xml")
+    Indexmap.create(configuration: config)
+    previous = config.storage.read(config.index_filename)
+    blobs = ActiveStorage::Blob.order(:id).pluck(:id)
+    records = SitemapArtifact.order(:id).pluck(:id)
+    service = ActiveStorage::Blob.service
+    uploaded = []
+    service.define_singleton_method(:upload) do |key, io, **options|
+      super(key, io, **options)
+      uploaded << key
+      raise IOError, "upload interrupted after object write" if uploaded.size == 2
+    end
+    assert_raises(IOError) { Indexmap.create(configuration: config) }
+    assert_equal previous, config.storage.read(config.index_filename)
+    assert_equal blobs, ActiveStorage::Blob.order(:id).pluck(:id)
+    assert_equal records, SitemapArtifact.order(:id).pluck(:id)
+    assert uploaded.none? { |key| service.exist?(key) }
+    assert Indexmap::Validator.new(configuration: config).validate!
+  ensure
+    service&.singleton_class&.remove_method(:upload)
+  end
+
+  def test_active_storage_cleans_a_losing_build_without_pruning
+    config = active_storage_configuration("race.xml")
+    storage = config.storage
+    baseline = ActiveStorage::Blob.pluck(:id)
+    lost_files = []
+    lost_keys = []
+    winner = nil
+    storage.define_singleton_method(:publish) do |source, **options|
+      lost_files.concat(options.fetch(:files).reject { |name| name == options.fetch(:root) })
+      source.define_singleton_method(:open) do |filename, &block|
+        if filename == options.fetch(:root)
+          lost_keys.concat(ActiveStorage::Blob.where.not(id: baseline).pluck(:key))
+          competing = Indexmap::Configuration.new
+          competing.base_url = config.base_url
+          competing.index_filename = config.index_filename
+          competing.atomic = true
+          competing.sections = config.sections
+          competing.storage = Indexmap::Storage::ActiveStorage.new(model: SitemapArtifact, public_url: config.base_url)
+          winner = Indexmap.create(configuration: competing)
+        end
+        super(filename, &block)
+      end
+      super(source, **options)
+    end
+    assert_raises(Indexmap::SupersededPublication) { Indexmap.create(configuration: config) }
+    assert_equal winner.size, ActiveStorage::Blob.where.not(id: baseline).count
+    assert lost_files.none? { |filename| storage.exist?(filename) }
+    assert lost_keys.none? { |key| ActiveStorage::Blob.service.exist?(key) }
+    assert winner.all? { |filename| storage.exist?(filename) }
+    assert Indexmap::Validator.new(configuration: config).validate!
+  end
+
+  def test_active_storage_does_not_activate_children_deleted_by_pruning
+    config = active_storage_configuration("prune-race.xml")
+    Indexmap.create(configuration: config)
+    previous = config.storage.read(config.index_filename)
+    blobs = ActiveStorage::Blob.pluck(:id).sort
+    config.storage.define_singleton_method(:publish) do |source, **options|
+      storage = self
+      source.define_singleton_method(:open) do |filename, &block|
+        storage.prune(root: options.fetch(:root), before: Time.now.utc + 1) if filename == options.fetch(:root)
+        super(filename, &block)
+      end
+      super(source, **options)
+    end
+    assert_raises(Indexmap::SupersededPublication) { Indexmap.create(configuration: config) }
+    assert_equal previous, config.storage.read(config.index_filename)
+    assert_equal blobs, ActiveStorage::Blob.pluck(:id).sort
+    assert Indexmap::Validator.new(configuration: config).validate!
+  end
+
+  def test_active_storage_preserves_a_commit_when_a_host_callback_raises
+    config = active_storage_configuration("callback.xml")
+    callback = proc do |record|
+      if record.filename == "callback.xml" && record.file.attached?
+        raise IOError, "host callback failed after commit"
+      end
+    end
+    SitemapArtifact.set_callback(:commit, :after, callback)
+    assert_raises(IOError) { Indexmap.create(configuration: config) }
+    assert config.storage.exist?(config.index_filename)
+    assert Indexmap::Validator.new(configuration: config).validate!
+  ensure
+    SitemapArtifact.skip_callback(:commit, :after, callback) if callback
+  end
+
+  def test_active_storage_reports_cleanup_failures_and_attempts_remaining_objects
+    config = active_storage_configuration("cleanup-error.xml")
+    Indexmap.create(configuration: config)
+    previous = config.storage.read(config.index_filename)
+    service = ActiveStorage::Blob.service
+    uploaded = []
+    attempted = []
+    service.define_singleton_method(:upload) do |key, io, **options|
+      super(key, io, **options)
+      uploaded << key
+      raise IOError, "original upload failure" if uploaded.size == 2
+    end
+    service.define_singleton_method(:delete) do |key|
+      attempted << key
+      raise IOError, "cleanup unavailable" if key == uploaded.first
+
+      super(key)
+    end
+    error = assert_raises(Indexmap::PublicationCleanupError) { Indexmap.create(configuration: config) }
+    assert_equal "original upload failure", error.cause.message
+    assert_equal [uploaded.first], error.failures.map { |failure| failure.fetch(:artifact) }
+    assert_equal uploaded.sort, attempted.sort
+    assert_equal previous, config.storage.read(config.index_filename)
+    assert Indexmap::Validator.new(configuration: config).validate!
+  ensure
+    if service
+      service.singleton_class.remove_method(:upload)
+      service.singleton_class.remove_method(:delete)
+      uploaded&.each { |key| service.delete(key) }
+    end
+  end
+
+  def test_active_storage_publishes_atomic_single_file_outputs
+    config = active_storage_configuration("single.xml")
+    config.format = :single_file
+    config.entries = [{loc: "https://example.com/page"}]
+    assert_equal ["single.xml"], Indexmap.create(configuration: config)
+    assert_equal ["single.xml"], Indexmap.create(configuration: config)
+    assert Indexmap::Validator.new(configuration: config).validate!
+  end
+
+  private
+
+  def active_storage_configuration(root)
+    Indexmap.configuration.tap do |config|
+      config.index_filename = root
+      config.storage = Indexmap::Storage::ActiveStorage.new(model: SitemapArtifact, public_url: config.base_url)
+    end
+  end
+
+  def wait_for(reader)
+    deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 30
+    log = +""
+    until yield
+      raise "Worker did not reach expected state: #{log[-2000, 2000]}" if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+
+      if IO.select([reader], nil, nil, 0.1)
+        chunk = reader.read_nonblock(4096, exception: false)
+        log << chunk if chunk.is_a?(String)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexmap
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -16,14 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '8.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '7.1'
+        version: '8.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -80,20 +86,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
-- !ruby/object:Gem::Dependency
-  name: railties
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '7.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -108,20 +100,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -136,20 +114,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.22'
-- !ruby/object:Gem::Dependency
-  name: standard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -175,9 +139,17 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- UPGRADE.md
+- docs/architecture.md
+- docs/deployment.md
+- docs/domain-terms.md
+- docs/dynamic-sitemaps.md
+- docs/search-engine-ping.md
+- docs/storage.md
 - lib/generators/indexmap/install_generator.rb
 - lib/generators/indexmap/templates/initializer.rb.tt
 - lib/indexmap.rb
+- lib/indexmap/artifact.rb
 - lib/indexmap/configuration.rb
 - lib/indexmap/creator.rb
 - lib/indexmap/entry.rb
@@ -188,29 +160,39 @@ files:
 - lib/indexmap/pinger/base.rb
 - lib/indexmap/pinger/google.rb
 - lib/indexmap/pinger/index_now.rb
+- lib/indexmap/publication.rb
+- lib/indexmap/rails.rb
 - lib/indexmap/railtie.rb
+- lib/indexmap/robots.rb
+- lib/indexmap/robots_controller.rb
 - lib/indexmap/section.rb
+- lib/indexmap/sitemaps_controller.rb
 - lib/indexmap/storage/active_storage.rb
-- lib/indexmap/storage/file.rb
 - lib/indexmap/storage/filesystem.rb
 - lib/indexmap/storage/memory.rb
 - lib/indexmap/task_runner.rb
 - lib/indexmap/tasks.rb
 - lib/indexmap/validator.rb
+- lib/indexmap/validator/duplicates.rb
 - lib/indexmap/version.rb
 - lib/indexmap/writer.rb
 - lib/tasks/indexmap_tasks.rake
 - test/indexmap/configuration_test.rb
+- test/indexmap/correctness_test.rb
 - test/indexmap/install_generator_test.rb
 - test/indexmap/lazy_loading_test.rb
 - test/indexmap/parser_test.rb
 - test/indexmap/pinger/google_test.rb
 - test/indexmap/pinger/index_now_test.rb
+- test/indexmap/publication_test.rb
+- test/indexmap/rails_test.rb
+- test/indexmap/robots_test.rb
+- test/indexmap/sitemaps_controller_test.rb
 - test/indexmap/storage_test.rb
 - test/indexmap/task_runner_test.rb
 - test/indexmap/validator_test.rb
 - test/indexmap/writer_test.rb
-- test/release_task_test.rb
+- test/integration/queue_checks.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/indexmap
 licenses:
@@ -219,8 +201,8 @@ metadata:
   homepage_uri: https://www.ethos-link.com/opensource/indexmap
   source_code_uri: https://github.com/ethos-link/indexmap
   bug_tracker_uri: https://github.com/ethos-link/indexmap/issues
-  changelog_uri: https://github.com/ethos-link/indexmap/blob/main/CHANGELOG.md
-  documentation_uri: https://github.com/ethos-link/indexmap/blob/main/README.md
+  changelog_uri: https://github.com/ethos-link/indexmap/blob/v0.10.0/CHANGELOG.md
+  documentation_uri: https://github.com/ethos-link/indexmap/blob/v0.10.0/README.md
   funding_uri: https://www.reviato.com/
   github_repo: ssh://github.com/ethos-link/indexmap
   allowed_push_host: https://rubygems.org
@@ -232,14 +214,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: '4.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Generate sitemap indexes and child sitemaps with plain Ruby
 test_files: []

data/lib/indexmap/storage/file.rb

@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Indexmap
-  module Storage
-    File = Struct.new(:filename, :body, :content_type) do
-      def basename
-        ::File.basename(filename)
-      end
-    end
-  end
-end

data/test/release_task_test.rb

@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-require "test_helper"
-require "rake"
-
-unless respond_to?(:release_version, true)
-  load File.expand_path("../Rakefile", __dir__)
-end
-
-class ReleaseTaskTest < Minitest::Test
-  def test_release_version_increments_patch_from_current_version
-    major, minor, patch = Indexmap::VERSION.split(".").map(&:to_i)
-
-    assert_equal "#{major}.#{minor}.#{patch + 1}", release_version("patch")
-  end
-
-  def test_release_version_accepts_explicit_semantic_version
-    assert_equal "0.3.0", release_version("0.3.0")
-  end
-
-  def test_validate_release_version_rejects_current_version
-    error = assert_raises(ArgumentError) do
-      validate_release_version!("0.2.7", "0.2.7")
-    end
-
-    assert_equal(
-      "Release version 0.2.7 must be newer than current version 0.2.7.",
-      error.message
-    )
-  end
-
-  def test_validate_release_version_rejects_existing_local_tag
-    @local_release_tag_exists = true
-    @remote_release_tag_exists = false
-
-    error = assert_raises(ArgumentError) do
-      validate_release_version!("0.2.8", "0.2.7")
-    end
-
-    assert_equal "Release tag v0.2.8 already exists locally.", error.message
-  end
-
-  def test_validate_release_version_rejects_existing_remote_tag
-    @local_release_tag_exists = false
-    @remote_release_tag_exists = true
-
-    error = assert_raises(ArgumentError) do
-      validate_release_version!("0.2.8", "0.2.7")
-    end
-
-    assert_equal "Release tag v0.2.8 already exists on origin.", error.message
-  end
-
-  def test_remote_release_tag_command_asks_git_to_fail_when_no_tag_matches
-    assert_equal(
-      "git ls-remote --exit-code --tags origin refs/tags/v0.2.8",
-      remote_release_tag_command("v0.2.8")
-    )
-  end
-
-  def test_changelog_command_prepends_the_next_release
-    assert_equal(
-      [
-        "git-cliff",
-        "-c",
-        "cliff.toml",
-        "--unreleased",
-        "--tag",
-        "v0.2.8",
-        "--prepend",
-        "CHANGELOG.md"
-      ],
-      changelog_command("0.2.8")
-    )
-  end
-
-  private
-
-  def local_release_tag_exists?(_tag)
-    @local_release_tag_exists || false
-  end
-
-  def remote_release_tag_exists?(_tag)
-    @remote_release_tag_exists || false
-  end
-end