indexmap 0.8.2 → 0.9.0

data/lib/indexmap/pinger/google.rb

@@ -4,12 +4,13 @@ require "google/apis/searchconsole_v1"
 require "googleauth"
 require "json"
 require "stringio"
+require "tmpdir"
 
 module Indexmap
   module Pinger
     class Google < Base
-      def initialize(configuration: Indexmap.configuration, service: nil, credentials_builder: nil)
-        super(configuration: configuration)
+      def initialize(configuration: Indexmap.configuration, storage: nil, service: nil, credentials_builder: nil)
+        super(configuration: configuration, storage: storage)
         @service = service
         @credentials_builder = credentials_builder
       end
@@ -80,7 +81,7 @@ module Indexmap
       end
 
       def authorizer
-        json_key = JSON.parse(google_configuration.credentials).to_json
+        json_key = JSON.generate(JSON.parse(google_configuration.credentials))
         scope = "https://www.googleapis.com/auth/webmasters"
 
         return credentials_builder.call(credentials: json_key, scope: scope) if credentials_builder
@@ -92,12 +93,15 @@ module Indexmap
       end
 
       def sitemap_url_count(files)
-        files.each_with_object(Set.new) do |sitemap_file, urls|
-          Parser.new(source: sitemap_file, storage: storage).entries.each do |entry|
-            loc = entry.loc.to_s.strip
-            urls.add(loc) unless loc.empty?
+        Dir.mktmpdir("indexmap-google-count") do |directory|
+          urls = Validator::Duplicates.new(directory: directory)
+          files.each do |sitemap_file|
+            Parser.new(source: sitemap_file, storage: storage).each_entry do |entry|
+              urls.add(JSON.generate(entry.loc))
+            end
           end
-        end.count
+          urls.unique_count
+        end
       end
 
       def summarize_results(results, url_count:)

data/lib/indexmap/pinger/index_now.rb

@@ -10,8 +10,8 @@ module Indexmap
     class IndexNow < Base
       KEY_FORMAT = /\A[a-f0-9]{32}\z/
 
-      def initialize(configuration: Indexmap.configuration, connection: nil)
-        super(configuration: configuration)
+      def initialize(configuration: Indexmap.configuration, storage: nil, connection: nil)
+        super(configuration: configuration, storage: storage)
         @connection = connection
       end
 
@@ -22,13 +22,9 @@ module Indexmap
           return {status: :skipped, reason: :missing_key}
         end
 
+        batch_size = max_urls_per_request
         entries = entries_to_ping
-        if entries.empty?
-          logger.debug("IndexNow: no URLs matched the current filter.")
-          return {status: :skipped, reason: :no_urls}
-        end
-
-        results = entries.each_slice(max_urls_per_request).map do |batch|
+        results = entries.each_slice(batch_size).map.with_index(1) do |batch, number|
           urls = batch.map(&:loc)
 
           if dry_run?
@@ -36,12 +32,55 @@ module Indexmap
             next({status: :dry_run, url_count: urls.count})
           end
 
-          submit_batch(api_key: api_key, urls: urls)
+          attempts = index_now_configuration.max_attempts || 1
+          max_wait = index_now_configuration.max_wait || 30
+          unless attempts.is_a?(Integer) && (1..5).cover?(attempts) && max_wait.is_a?(Numeric) && (0..60).cover?(max_wait)
+            raise ConfigurationError, "IndexNow retries require 1..5 attempts and 0..60 seconds max_wait"
+          end
+          result = nil
+          attempts.times do |attempt|
+            yield({event: :attempt_started, batch: number, attempt: attempt + 1, urls: urls}) if block_given?
+            result = submit_batch(api_key: api_key, urls: urls)
+            yield(result.merge(event: :attempt_finished, batch: number, attempt: attempt + 1, urls: urls)) if block_given?
+            retryable = result[:reason] == :unknown_delivery || [429, 500, 502, 503, 504].include?(result[:status_code])
+            break unless retryable && attempt + 1 < attempts
+
+            delay = result[:retry_after] || [2**attempt + rand, max_wait].min
+            break if delay > max_wait
+
+            sleep(delay)
+          end
+          result
         end
 
+        return {status: :skipped, reason: :no_urls} if results.empty?
+
         summarize_results(results)
       end
 
+      def plan
+        return enum_for(:plan) unless block_given?
+
+        cutoff = since_cutoff
+        batch_size = max_urls_per_request
+        missing_key = read_api_key.nil?
+        selected_count = 0
+        each_selection(cutoff: cutoff) do |entry, reason, selected|
+          selected_count += 1 if selected
+          yield({
+            loc: entry.loc,
+            lastmod: entry.lastmod,
+            source_sitemap: entry.source_sitemap,
+            index_filename: configuration.index_filename,
+            cutoff: cutoff&.iso8601,
+            reason: reason,
+            selected: selected,
+            batch: selected ? ((selected_count - 1) / batch_size) + 1 : nil,
+            missing_key: missing_key
+          })
+        end
+      end
+
       def write_key_file(key: index_now_configuration.key, filename: nil)
         key = normalized_configured_key(key)
         return if key.empty?
@@ -72,34 +111,26 @@ module Indexmap
         configuration.index_now
       end
 
-      def sitemap_files
-        files = super
-        return files if files.one?
-
-        child_files = files.reject { |file| file == configuration.index_filename }
-        child_files.empty? ? files : child_files
-      end
-
       def entries_to_ping
-        cutoff = since_cutoff
-        unless cutoff
-          logger.debug("IndexNow: no cutoff provided, submitting all sitemap URLs.")
-          return current_entries.values
-        end
-
-        logger.debug { "IndexNow: submitting sitemap URLs with lastmod >= #{cutoff.iso8601}." }
-
-        current_entries.values.select do |entry|
-          lastmod_after_cutoff?(entry, cutoff) || entry.lastmod.to_s.strip.empty?
-        end
+        enum_for(:each_selection, cutoff: since_cutoff).lazy.filter_map do |entry, _reason, selected|
+          entry if selected
+        end.eager
       end
 
-      def current_entries
-        sitemap_files.each_with_object({}) do |sitemap_file, entries|
-          Parser.new(source: sitemap_file, storage: storage).entries.each do |entry|
-            next if entry.loc.to_s.strip.empty?
-
-            entries[entry.loc] = entry
+      def each_selection(cutoff:)
+        sitemap_files.each do |filename|
+          validator = Validator.new(configuration: configuration, filename: filename, storage: storage)
+          validator.each_entry do |entry|
+            reason = if cutoff.nil?
+              :all
+            elsif entry.lastmod.to_s.strip.empty?
+              :missing_lastmod
+            elsif lastmod_after_cutoff?(entry, cutoff)
+              :recent
+            else
+              :before_cutoff
+            end
+            yield entry, reason, reason != :before_cutoff
           end
         end
       end
@@ -126,23 +157,20 @@ module Indexmap
       end
 
       def lastmod_after_cutoff?(entry, cutoff)
-        lastmod = entry_lastmod(entry)
-        return false unless lastmod
-
-        lastmod >= cutoff
-      end
-
-      def entry_lastmod(entry)
-        return if entry.lastmod.to_s.strip.empty?
-
-        Time.iso8601(entry.lastmod.to_s).utc
-      rescue ArgumentError
-        logger.debug { "IndexNow: skipping invalid sitemap lastmod #{entry.lastmod.inspect} for #{entry.loc}" }
-        nil
+        if entry.lastmod.to_s.match?(/\A\d{4}-\d{2}-\d{2}\z/)
+          Date.iso8601(entry.lastmod) >= cutoff.to_date
+        else
+          Time.iso8601(entry.lastmod).utc >= cutoff
+        end
       end
 
       def max_urls_per_request
-        ENV.fetch("INDEXNOW_MAX_URLS_PER_REQUEST", index_now_configuration.max_urls_per_request).to_i
+        value = ENV.fetch("INDEXNOW_MAX_URLS_PER_REQUEST") { index_now_configuration.max_urls_per_request }
+        size = Integer(value.to_s, 10, exception: false)
+        unless size && (1..10_000).cover?(size)
+          raise ConfigurationError, "IndexNow batch size must be an integer from 1 through 10,000"
+        end
+        size
       end
 
       def submit_batch(api_key:, urls:)
@@ -152,20 +180,37 @@ module Indexmap
 
         response = index_now_connection.post("/indexnow") do |request|
           request.headers["Content-Type"] = "application/json"
-          request.body = payload.to_json
+          request.body = JSON.generate(payload)
         end
 
         if response.success?
           logger.debug { "Successfully pinged IndexNow with #{urls.count} URLs." }
-          {status: :submitted, url_count: urls.count}
+          {status: :submitted, url_count: urls.count, status_code: response.status,
+           receipt: (response.status == 202) ? :validation_pending : :received}
         else
-          logger.debug { "Failed to ping IndexNow. Status: #{response.status}, Body: #{response.body}" }
-          {status: :failed, url_count: urls.count, status_code: response.status, body: response.body}
+          logger.debug { "Failed to ping IndexNow. Status: #{response.status}" }
+          {status: :failed, url_count: urls.count, status_code: response.status,
+           body: response.body.to_s.encode("UTF-8", invalid: :replace, undef: :replace).gsub(api_key, "[FILTERED]").slice(0, 1024),
+           retry_after: retry_after(response.headers["retry-after"])}
         end
+      rescue Faraday::TimeoutError, Faraday::ConnectionFailed
+        {status: :failed, url_count: urls.count, reason: :unknown_delivery}
+      end
+
+      def retry_after(value)
+        return if value.to_s.empty?
+
+        seconds = Integer(value.to_s, exception: false)
+        seconds ||= Time.httpdate(value).utc - Time.now.utc
+        [seconds, 0].max
+      rescue ArgumentError
+        nil
       end
 
       def index_now_connection
         @index_now_connection ||= connection || Faraday.new(url: index_now_configuration.endpoint) do |faraday|
+          faraday.options.open_timeout = 5
+          faraday.options.timeout = 20
           faraday.request :json
         end
       end

data/lib/indexmap/publication.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "digest"
+require "uri"
+
+module Indexmap
+  class Publication
+    def self.immutable?(filename)
+      File.basename(filename).match?(/\Asitemap-[a-f0-9]{16}-[a-f0-9]{32}-[1-9]\d*-[1-9]\d*\.xml\z/)
+    end
+
+    def initialize(root:)
+      @root = root
+      @prefix = File.join(File.dirname(root), "sitemap-#{Digest::SHA256.hexdigest(root)[0, 16]}-").delete_prefix("./")
+    end
+
+    attr_reader :root, :prefix
+
+    def filename(generation:, section:, part:)
+      "#{prefix}#{generation}-#{section}-#{part}.xml"
+    end
+
+    def generation(filename)
+      filename[/\A#{Regexp.escape(prefix)}([a-f0-9]{32})-[1-9]\d*-[1-9]\d*\.xml\z/, 1]
+    end
+
+    def child?(filename)
+      !generation(filename).nil?
+    end
+
+    def archive(generation)
+      "#{prefix}#{generation}-index.xml"
+    end
+
+    def validate_files!(files)
+      children = files.reject { |filename| filename == root }
+      unless files.count(root) == 1 && files.uniq.size == files.size &&
+          children.all? { |filename| child?(filename) } &&
+          children.map { |filename| generation(filename) }.uniq.size <= 1
+        raise ConfigurationError, "Publication must contain its root and one immutable generation"
+      end
+    end
+
+    def current_generation(storage)
+      return unless storage.exist?(root)
+
+      metadata = storage.metadata(root)
+      return metadata[:generation] if metadata.key?(:generation)
+
+      result = nil
+      storage.open(root) do |io|
+        Nokogiri::XML::Reader.from_io(io, nil, nil, Nokogiri::XML::ParseOptions::NONET).each do |node|
+          if node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT && node.local_name == "loc"
+            element = Nokogiri::XML(node.outer_xml) { |options| options.strict.nonet }
+            path = URI.parse(element.root.text).path
+            candidate = File.join(File.dirname(root), File.basename(path)).delete_prefix("./")
+            result = generation(candidate)
+            break
+          end
+        end
+      end
+      result
+    end
+
+    def prune(storage, before:)
+      return [] unless storage.exist?(root)
+
+      current = current_generation(storage)
+      files = storage.list(prefix: prefix, suffix: ".xml")
+      deleted = []
+      files.each do |filename|
+        match = filename.match(/\A#{Regexp.escape(prefix)}([a-f0-9]{32})-index\.xml\z/)
+        next unless match && match[1] != current && storage.metadata(filename).fetch(:last_modified) < before
+
+        block_given? ? yield(filename) : storage.delete(filename)
+        deleted << filename
+      end
+      files.each do |filename|
+        version = generation(filename)
+        next unless version && version != current && !storage.exist?(archive(version))
+        next unless storage.metadata(filename).fetch(:last_modified) < before
+
+        block_given? ? yield(filename) : storage.delete(filename)
+        deleted << filename
+      end
+      deleted
+    end
+  end
+end

data/lib/indexmap/robots.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Indexmap
+  class Robots
+    attr_writer :indexable, :groups, :sitemaps
+
+    def initialize(configuration:)
+      @configuration = configuration
+      @indexable = false
+      @groups = [{user_agent: "*", allow: ["/"]}]
+    end
+
+    def body
+      return "User-agent: *\nDisallow: /\n" unless resolve(@indexable) == true
+
+      groups = Array(resolve(@groups))
+      raise ConfigurationError, "Robots groups must not be empty" if groups.empty?
+
+      lines = groups.flat_map do |group|
+        unknown = group.keys - %i[user_agent allow disallow]
+        raise ConfigurationError, "Unknown robots group keys: #{unknown.join(", ")}" unless unknown.empty?
+
+        agents = Array(group.fetch(:user_agent))
+        raise ConfigurationError, "Robots user agents must not be empty" if agents.empty?
+
+        agents.map { |agent| "User-agent: #{value(agent)}" } +
+          Array(group[:allow]).map { |path| "Allow: #{path_value(path)}" } +
+          Array(group[:disallow]).map { |path| "Disallow: #{path_value(path)}" } + [""]
+      end
+      sitemaps = resolve(@sitemaps) || [File.join(@configuration.base_url.to_s, @configuration.index_filename.to_s)]
+      Array(sitemaps).each do |location|
+        location = value(location)
+        uri = URI.parse(location)
+        unless uri.is_a?(URI::HTTP) && uri.host && !uri.userinfo && !uri.query && !uri.fragment
+          raise ConfigurationError, "Robots sitemap must be an absolute HTTP(S) URL"
+        end
+        lines << "Sitemap: #{location}"
+      end
+      lines.join("\n") + "\n"
+    rescue URI::InvalidURIError
+      raise ConfigurationError, "Robots sitemap must be an absolute HTTP(S) URL"
+    end
+
+    private
+
+    def resolve(value)
+      value.respond_to?(:call) ? value.call : value
+    end
+
+    def value(input)
+      text = input.to_s
+      if text.empty? || text.match?(/[\x00-\x1f\x7f#]/)
+        raise ConfigurationError, "Robots values must be nonempty and contain no controls or comments"
+      end
+      text
+    end
+
+    def path_value(input)
+      path = value(input)
+      raise ConfigurationError, "Robots paths must start with /" unless path.start_with?("/")
+
+      path
+    end
+  end
+end

data/lib/indexmap/robots_controller.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "indexmap"
+require "action_controller"
+
+class Indexmap::RobotsController < ActionController::Base
+  def show
+    body = Indexmap.configuration.robots.body
+    expires_in 1.hour, public: true
+    fresh_when etag: body, public: true, template: false
+    render plain: body, content_type: "text/plain" unless performed?
+  end
+end

data/lib/indexmap/sitemaps_controller.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "indexmap"
+require "rails"
+require "action_controller"
+
+class Indexmap::SitemapsController < ActionController::Base
+  def show
+    configuration = Indexmap.configuration
+    storage = configuration.storage
+    unless storage.respond_to?(:metadata) && storage.respond_to?(:open)
+      raise Indexmap::ConfigurationError, "Sitemap delivery requires storage metadata and open capabilities"
+    end
+    path = request.path_parameters
+    filename = [path[:prefix], path.fetch(:filename)].compact.join("/")
+    allowed = configuration.output_names.any? do |name|
+      output = configuration.output_for(name)
+      root = output.index_filename
+      next true if filename == root
+      next false unless output.atomic
+
+      publication = Indexmap::Publication.new(root: root)
+      version = publication.generation(filename)
+      next false unless version
+      next true if storage.exist?(publication.archive(version))
+      next false unless storage.exist?(root)
+
+      metadata = storage.metadata(root)
+      active = Rails.cache.fetch(["indexmap-generation", storage.public_url(root), metadata.fetch(:etag)], expires_in: 1.hour) do
+        publication.current_generation(storage)
+      end
+      version == active
+    end
+    return head :not_found unless allowed && storage.exist?(filename)
+
+    metadata = storage.metadata(filename)
+    if Indexmap::Publication.immutable?(filename)
+      expires_in 1.year, public: true, immutable: true
+    else
+      expires_in 10.minutes, public: true, stale_while_revalidate: 1.hour
+    end
+    fresh_when etag: metadata.fetch(:etag), last_modified: metadata.fetch(:last_modified), public: true, template: false
+    return if performed?
+
+    response.content_type = "application/xml"
+    response_body = Enumerator.new do |stream|
+      storage.open(filename) do |io|
+        while (chunk = io.read(64 * 1024))
+          stream << chunk
+        end
+      end
+    end
+    self.response_body = response_body
+  rescue Errno::ENOENT
+    head :not_found
+  end
+end

data/lib/indexmap/storage/active_storage.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "stringio"
+require "securerandom"
 require "uri"
 
 module Indexmap
@@ -17,6 +18,9 @@ module Indexmap
       end
 
       def write(filename, body, content_type: nil)
+        if Publication.immutable?(filename) && exist?(filename)
+          raise ConfigurationError, "Immutable sitemap already exists: #{filename}"
+        end
         attachment_content_type = content_type || default_content_type
         record = find_or_initialize(filename)
         record.save! unless record.persisted?
@@ -26,7 +30,7 @@ module Indexmap
           content_type: attachment_content_type
         )
 
-        File.new(filename: filename, body: body.to_s, content_type: attachment_content_type)
+        Artifact.new(filename: filename, body: body.to_s, content_type: attachment_content_type)
       end
 
       def read(filename)
@@ -42,6 +46,8 @@ module Indexmap
 
       def list(prefix: nil, suffix: nil)
         relation = relation_for(prefix: prefix, suffix: suffix)
+        scope = "with_attached_#{attachment}"
+        relation = relation.public_send(scope) if relation.respond_to?(scope)
 
         relation.filter_map do |record|
           filename = record.public_send(filename_column).to_s
@@ -62,6 +68,102 @@ module Indexmap
         URI.join("#{public_url_base}/", filename).to_s
       end
 
+      def write_io(filename, io, content_type: nil)
+        if Publication.immutable?(filename) && exist?(filename)
+          raise ConfigurationError, "Immutable sitemap already exists: #{filename}"
+        end
+        record = find_or_initialize(filename)
+        record.save! unless record.persisted?
+        record.public_send(attachment).attach(io: io, filename: filename,
+          content_type: content_type || default_content_type)
+      end
+
+      def open(filename, &block)
+        blob_for(filename).open(&block)
+      end
+
+      def metadata(filename)
+        record = find_record(filename)
+        raise Errno::ENOENT, filename unless attached?(record)
+
+        blob = record.public_send(attachment).blob
+        {etag: blob.checksum, bytes: blob.byte_size, last_modified: record.updated_at,
+         generation: blob.metadata["indexmap_generation"]}
+      end
+
+      def revision(root)
+        exist?(root) ? blob_for(root).id.to_s : nil
+      end
+
+      def publish(source, files:, root:, expected:)
+        uploads = []
+        created_records = []
+        root_blob = nil
+        build = SecureRandom.hex(16)
+        unique = model.connection.indexes(model.table_name).any? do |index|
+          index.unique && index.columns == [filename_column.to_s] && index.where.nil?
+        end
+        raise ConfigurationError, "Atomic Active Storage requires a unique filename index" unless unique
+
+        check_revision!(root, expected)
+        publication = Publication.new(root: root)
+        publication.validate_files!(files)
+        children = files.reject { |filename| filename == root }
+        children.each do |filename|
+          raise ConfigurationError, "Immutable sitemap already exists: #{filename}" if exist?(filename)
+        end
+        record = model.create_or_find_by!(filename_column => root)
+        [*children, root].each do |filename|
+          source.open(filename) do |io|
+            check_revision!(root, expected)
+            key = SecureRandom.hex(28)
+            uploads << key
+            metadata = {indexmap_build: build}
+            metadata[:indexmap_generation] = children.empty? ? nil : publication.generation(children.first) if filename == root
+            blob = ::ActiveStorage::Blob.create_and_upload!(key: key, io: io, filename: ::File.basename(filename),
+              content_type: default_content_type, identify: false, metadata: metadata)
+            if filename == root
+              root_blob = blob
+            else
+              child = model.create!(filename_column => filename)
+              created_records << child.id
+              child.public_send(attachment).attach(blob)
+            end
+          end
+        end
+        record.with_lock do
+          current = record.public_send(attachment)
+          actual = current.attached? ? current.blob.id.to_s : nil
+          raise SupersededPublication, "Sitemap changed while building: #{root}" unless actual == expected
+          present = model.where(filename_column => children).joins(:"#{attachment}_attachment").distinct.count(filename_column)
+          unless present == children.size
+            raise SupersededPublication, "Sitemap children removed while building: #{root}"
+          end
+
+          if current.attached? && (version = publication.current_generation(self))
+            archived = model.create_or_find_by!(filename_column => publication.archive(version))
+            archived.public_send(attachment).attach(current.blob)
+          end
+          current.attach(root_blob)
+        end
+      ensure
+        # Also recognize a committed root when a host after_commit callback raises.
+        unless root_blob&.attachments&.exists?
+          cleanup_uploads(created_records || [], uploads || [], build: build)
+        end
+      end
+
+      def prune(root:, before:)
+        record = find_record(root)
+        return [] unless record
+
+        record.with_lock do
+          Publication.new(root: root).prune(self, before: before) do |filename|
+            find_record(filename).destroy!
+          end
+        end
+      end
+
       def inspect
         "#<#{self.class.name} model=#{model}>"
       end
@@ -70,6 +172,35 @@ module Indexmap
 
       attr_reader :model, :filename_column, :attachment, :default_content_type
 
+      def check_revision!(root, expected)
+        raise SupersededPublication, "Sitemap changed while building: #{root}" unless revision(root) == expected
+      end
+
+      def cleanup_uploads(records, keys, build:)
+        failures = []
+        records.each do |id|
+          model.find_by(model.primary_key => id)&.destroy!
+        rescue => error
+          failures << {artifact: "#{model.name}:#{id}", error: error}
+        end
+        keys.each do |key|
+          blob = ::ActiveStorage::Blob.find_by(key: key)
+          if blob && blob.metadata["indexmap_build"] == build && !blob.attachments.exists?
+            blob.purge
+          end
+        rescue => error
+          failures << {artifact: key, error: error}
+        end
+        raise PublicationCleanupError.new(failures) unless failures.empty?
+      end
+
+      def blob_for(filename)
+        record = find_record(filename)
+        raise Errno::ENOENT, filename unless attached?(record)
+
+        record.public_send(attachment).blob
+      end
+
       def find_or_initialize(filename)
         model.find_or_initialize_by(filename_column => filename)
       end