indexmap 0.6.0 → 0.7.1

data/lib/indexmap/parser.rb

@@ -8,11 +8,11 @@ module Indexmap
   class Parser
     Entry = Struct.new(:loc, :lastmod, :source_sitemap, keyword_init: true)
 
-    def initialize(path: default_path, rebase_remote_children: false, index_filename: Indexmap.configuration.index_filename, public_path: Indexmap.configuration.public_path)
-      @source = path.to_s
+    def initialize(source: nil, rebase_remote_children: false, index_filename: Indexmap.configuration.index_filename, storage: Indexmap.configuration.storage)
+      @source = (source || index_filename).to_s
       @rebase_remote_children = rebase_remote_children
       @index_filename = index_filename
-      @public_path = public_path
+      @storage = storage
     end
 
     def entries(reset: false)
@@ -58,11 +58,7 @@ module Indexmap
 
     private
 
-    attr_reader :index_filename, :public_path
-
-    def default_path
-      Indexmap::Path.existing_public_path(public_path: public_path, index_filename: index_filename)
-    end
+    attr_reader :index_filename, :storage
 
     def parse_source(source, visited:)
       normalized_source = normalize_source(source)
@@ -105,12 +101,21 @@ module Indexmap
         end
       elsif remote_source?(loc)
         uri = URI.parse(loc)
-        File.join(File.dirname(parent_source), File.basename(uri.path))
+        normalize_local_source(uri.path)
       else
-        File.expand_path(loc, File.dirname(parent_source))
+        resolve_local_child_sitemap(parent_source, loc)
       end
     rescue URI::InvalidURIError
-      File.expand_path(loc, File.dirname(parent_source))
+      resolve_local_child_sitemap(parent_source, loc)
+    end
+
+    def resolve_local_child_sitemap(parent_source, loc)
+      if loc.start_with?("/")
+        normalize_local_source(loc)
+      else
+        parent_directory = File.dirname(parent_source)
+        normalize_local_source((parent_directory == ".") ? loc : File.join(parent_directory, loc))
+      end
     end
 
     def remote_child_source(parent_uri, loc)
@@ -130,12 +135,19 @@ module Indexmap
       if remote_source?(source)
         URI.parse(source).to_s
       else
-        Pathname(source).expand_path.to_s
+        normalize_local_source(source)
       end
     rescue URI::InvalidURIError
       nil
     end
 
+    def normalize_local_source(source)
+      normalized = Pathname(source.to_s).cleanpath.to_s.sub(%r{\A/+}, "")
+      return if normalized.empty? || normalized == ".." || normalized.start_with?("../")
+
+      normalized
+    end
+
     def remote_source?(value)
       uri = URI.parse(value.to_s)
       uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
@@ -146,8 +158,8 @@ module Indexmap
     def read_source(source)
       if remote_source?(source)
         fetch_remote_source(source)
-      elsif File.exist?(source)
-        File.read(source, encoding: "UTF-8")
+      elsif storage.exist?(source)
+        storage.read(source)
       end
     end
 

data/lib/indexmap/pinger/base.rb

@@ -46,8 +46,12 @@ module Indexmap
         hostname.sub(/\Awww\./, "")
       end
 
+      def storage
+        configuration.storage
+      end
+
       def sitemap_files
-        Dir.glob(configuration.public_path.join("sitemap*.xml")).sort
+        storage.list(prefix: "sitemap", suffix: ".xml")
       end
 
       def ping_sitemap(_sitemap_file)

data/lib/indexmap/pinger/google.rb

@@ -36,7 +36,7 @@ module Indexmap
       end
 
       def ping_sitemap(sitemap_file)
-        sitemap_url = URI.join(host, File.basename(sitemap_file)).to_s
+        sitemap_url = storage.public_url(sitemap_file)
 
         unless authorized?
           logger.debug("Google Search Console does not have access to the site: #{root_domain}")
@@ -93,7 +93,7 @@ module Indexmap
 
       def sitemap_url_count(files)
         files.each_with_object(Set.new) do |sitemap_file, urls|
-          Parser.new(path: sitemap_file).entries.each do |entry|
+          Parser.new(source: sitemap_file, storage: storage).entries.each do |entry|
             loc = entry.loc.to_s.strip
             urls.add(loc) unless loc.empty?
           end

data/lib/indexmap/pinger/index_now.rb

@@ -42,14 +42,15 @@ module Indexmap
         summarize_results(results)
       end
 
-      def write_key_file(key: index_now_configuration.key, path: nil)
+      def write_key_file(key: index_now_configuration.key, filename: nil)
         key = normalized_configured_key(key)
         return if key.empty?
 
-        path ||= index_now_configuration.key_path(public_path: configuration.public_path, key: key)
-        FileUtils.mkdir_p(path.dirname)
-        File.write(path, key)
-        path
+        filename ||= index_now_configuration.key_filename(key: key)
+        return filename if valid_key_file?(filename)
+
+        storage.write(filename, key, content_type: "text/plain")
+        filename
       end
 
       def ensure_key_file
@@ -60,7 +61,7 @@ module Indexmap
         return existing_path if existing_path
 
         key = generated_key
-        write_key_file(key: key, path: configuration.public_path.join("#{key}.txt"))
+        write_key_file(key: key, filename: "#{key}.txt")
       end
 
       private
@@ -75,7 +76,7 @@ module Indexmap
         files = super
         return files if files.one?
 
-        child_files = files.reject { |file| File.basename(file) == configuration.index_filename }
+        child_files = files.reject { |file| file == configuration.index_filename }
         child_files.empty? ? files : child_files
       end
 
@@ -95,7 +96,7 @@ module Indexmap
 
       def current_entries
         sitemap_files.each_with_object({}) do |sitemap_file, entries|
-          Parser.new(path: sitemap_file).entries.each do |entry|
+          Parser.new(source: sitemap_file, storage: storage).entries.each do |entry|
             next if entry.loc.to_s.strip.empty?
 
             entries[entry.loc] = entry
@@ -177,36 +178,32 @@ module Indexmap
         configured_key = normalized_configured_key(index_now_configuration.key)
         return configured_key unless configured_key.empty?
 
-        existing_key_file&.read
+        storage.read(existing_key_file) if existing_key_file
       end
 
       def existing_key_file
-        configured_path = index_now_configuration.key_path(public_path: configuration.public_path)
-        return configured_path if valid_key_file?(configured_path)
+        configured_filename = index_now_configuration.key_filename
+        return configured_filename if valid_key_file?(configured_filename)
 
-        configuration.public_path.glob("*.txt").sort.find { |file| valid_key_file?(file) }
+        storage.list(suffix: ".txt").find { |filename| valid_key_file?(filename) }
       end
 
       def key_location(api_key:)
-        path = index_now_configuration.key_path(public_path: configuration.public_path, key: api_key) || existing_key_file
-        return unless path
-
-        public_path = configuration.public_path.expand_path
-        key_path = path.expand_path
-        relative_path = key_path.relative_path_from(public_path)
+        filename = index_now_configuration.key_filename(key: api_key) || existing_key_file
+        return unless filename
 
-        URI.join("#{host}/", relative_path.to_s).to_s
+        storage.public_url(filename)
       rescue ArgumentError
         nil
       end
 
-      def valid_key_file?(path)
-        return false unless path&.file?
+      def valid_key_file?(filename)
+        return false unless filename && storage.exist?(filename)
 
-        filename = path.basename(".txt").to_s
-        return false unless filename.match?(KEY_FORMAT)
+        key = File.basename(filename, ".txt").to_s
+        return false unless key.match?(KEY_FORMAT)
 
-        path.read == filename
+        storage.read(filename) == key
       end
 
       def generated_key

data/lib/indexmap/storage/active_storage.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "uri"
+
+module Indexmap
+  module Storage
+    class ActiveStorage
+      DEFAULT_CONTENT_TYPE = "application/xml"
+
+      def initialize(model:, public_url:, filename_column: :filename, attachment: :file, content_type: DEFAULT_CONTENT_TYPE)
+        @model = model
+        @public_url_base = public_url
+        @filename_column = filename_column
+        @attachment = attachment
+        @default_content_type = content_type
+      end
+
+      def write(filename, body, content_type: nil)
+        attachment_content_type = content_type || default_content_type
+        record = find_or_initialize(filename)
+        record.save! unless record.persisted?
+        record.public_send(attachment).attach(
+          io: StringIO.new(body.to_s),
+          filename: filename,
+          content_type: attachment_content_type
+        )
+
+        File.new(filename: filename, body: body.to_s, content_type: attachment_content_type)
+      end
+
+      def read(filename)
+        record = find_record(filename)
+        return unless attached?(record)
+
+        record.public_send(attachment).download
+      end
+
+      def exist?(filename)
+        attached?(find_record(filename))
+      end
+
+      def list(prefix: nil, suffix: nil)
+        relation = relation_for(prefix: prefix, suffix: suffix)
+
+        relation.filter_map do |record|
+          filename = record.public_send(filename_column).to_s
+          next if prefix && !filename.start_with?(prefix)
+          next if suffix && !filename.end_with?(suffix)
+          next unless attached?(record)
+
+          filename
+        end.sort
+      end
+
+      def delete(filename)
+        record = find_record(filename)
+        record&.public_send(attachment)&.purge
+      end
+
+      def public_url(filename)
+        URI.join("#{public_url_base}/", filename).to_s
+      end
+
+      def inspect
+        "#<#{self.class.name} model=#{model}>"
+      end
+
+      private
+
+      attr_reader :model, :filename_column, :attachment, :default_content_type
+
+      def find_or_initialize(filename)
+        model.find_or_initialize_by(filename_column => filename)
+      end
+
+      def find_record(filename)
+        model.find_by(filename_column => filename)
+      end
+
+      def relation_for(prefix:, suffix:)
+        if !model.respond_to?(:where) || (!prefix && !suffix)
+          return model.all
+        end
+
+        column = model.connection.quote_column_name(filename_column)
+        if prefix && suffix
+          model.where("#{column} LIKE ?", "#{prefix}%#{suffix}")
+        elsif prefix
+          model.where("#{column} LIKE ?", "#{prefix}%")
+        else
+          model.where("#{column} LIKE ?", "%#{suffix}")
+        end
+      end
+
+      def attached?(record)
+        record&.public_send(attachment)&.attached?
+      end
+
+      def public_url_base
+        @public_url_base.to_s.delete_suffix("/")
+      end
+    end
+  end
+end

data/lib/indexmap/storage/file.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Indexmap
+  module Storage
+    File = Struct.new(:filename, :body, :content_type, keyword_init: true) do
+      def basename
+        ::File.basename(filename)
+      end
+    end
+  end
+end

data/lib/indexmap/storage/filesystem.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Indexmap
+  module Storage
+    class Filesystem
+      DEFAULT_CONTENT_TYPE = "application/xml"
+
+      def initialize(path:, public_url: nil)
+        @path = Pathname(path)
+        @public_url_base = public_url
+      end
+
+      def write(filename, body, content_type: DEFAULT_CONTENT_TYPE)
+        target = path_for(filename)
+        target.dirname.mkpath
+        target.write(body.to_s)
+
+        File.new(
+          filename: normalize_filename(filename),
+          body: body.to_s,
+          content_type: content_type || DEFAULT_CONTENT_TYPE
+        )
+      end
+
+      def read(filename)
+        path_for(filename).read(encoding: "UTF-8")
+      end
+
+      def exist?(filename)
+        path_for(filename).file?
+      end
+
+      def list(prefix: nil, suffix: nil)
+        path.glob("**/*").select(&:file?).filter_map do |file|
+          filename = file.relative_path_from(path).to_s
+          next if prefix && !filename.start_with?(prefix)
+          next if suffix && !filename.end_with?(suffix)
+
+          filename
+        end.sort
+      end
+
+      def delete(filename)
+        path_for(filename).delete if exist?(filename)
+      end
+
+      def public_url(filename)
+        return normalize_filename(filename) if public_url_base.to_s.strip.empty?
+
+        URI.join("#{public_url_base.to_s.delete_suffix("/")}/", normalize_filename(filename)).to_s
+      end
+
+      def inspect
+        "#<#{self.class.name} path=#{path}>"
+      end
+
+      private
+
+      attr_reader :path, :public_url_base
+
+      def path_for(filename)
+        path.join(normalize_filename(filename))
+      end
+
+      def normalize_filename(filename)
+        normalized = Pathname(filename.to_s).cleanpath
+        if normalized.absolute? || normalized.to_s.start_with?("../") || normalized.to_s == ".."
+          raise ArgumentError, "Storage filename must be relative: #{filename.inspect}"
+        end
+
+        normalized.to_s
+      end
+    end
+  end
+end

data/lib/indexmap/storage/memory.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Indexmap
+  module Storage
+    class Memory
+      DEFAULT_CONTENT_TYPE = "application/xml"
+
+      def initialize(files = [], public_url: nil)
+        @files = {}
+        @public_url_base = public_url
+        Array(files).each do |file|
+          write(file.filename, file.body, content_type: file.content_type)
+        end
+      end
+
+      def write(filename, body, content_type: DEFAULT_CONTENT_TYPE)
+        normalized = normalize_filename(filename)
+        files[normalized] = File.new(
+          filename: normalized,
+          body: body.to_s,
+          content_type: content_type || DEFAULT_CONTENT_TYPE
+        )
+      end
+
+      def read(filename)
+        files.fetch(normalize_filename(filename)).body
+      end
+
+      def exist?(filename)
+        files.key?(normalize_filename(filename))
+      end
+
+      def list(prefix: nil, suffix: nil)
+        files.keys.select do |filename|
+          (prefix.nil? || filename.start_with?(prefix)) &&
+            (suffix.nil? || filename.end_with?(suffix))
+        end.sort
+      end
+
+      def delete(filename)
+        files.delete(normalize_filename(filename))
+      end
+
+      def public_url(filename)
+        return normalize_filename(filename) if public_url_base.to_s.strip.empty?
+
+        URI.join("#{public_url_base.to_s.delete_suffix("/")}/", normalize_filename(filename)).to_s
+      end
+
+      private
+
+      attr_reader :files, :public_url_base
+
+      def normalize_filename(filename)
+        filename.to_s
+      end
+    end
+  end
+end

data/lib/indexmap/task_runner.rb

@@ -10,15 +10,20 @@ module Indexmap
 
     def create
       written_files = Indexmap.create(configuration: configuration)
-      index_now_key_path = write_index_now_key
+      index_now_key_filename = write_index_now_key if configuration.index_now.write_key_file?
       configuration.run_after_create_callbacks
 
-      {files: written_files.map(&:to_s), written_files: written_files, index_now_key_path: index_now_key_path}
+      {
+        files: written_files.map(&:to_s),
+        written_files: written_files,
+        sitemaps: sitemap_details(written_files),
+        index_now_key_filename: index_now_key_filename
+      }
     end
 
     def format
-      sitemap_files.each do |file_path|
-        content = File.read(file_path)
+      sitemap_files.each do |filename|
+        content = storage.read(filename)
         document = Nokogiri::XML(
           content,
           nil,
@@ -27,7 +32,7 @@ module Indexmap
         )
         save_options = Nokogiri::XML::Node::SaveOptions::FORMAT | Nokogiri::XML::Node::SaveOptions::AS_XML
 
-        File.write(file_path, document.to_xml(indent: 2, save_with: save_options))
+        storage.write(filename, document.to_xml(indent: 2, save_with: save_options), content_type: "application/xml")
       end
 
       sitemap_files
@@ -45,8 +50,8 @@ module Indexmap
       pinger.write_key_file
     end
 
-    def public_path
-      default_output.public_path
+    def storage
+      configuration.storage
     end
 
     private
@@ -58,7 +63,29 @@ module Indexmap
     end
 
     def sitemap_files
-      Dir.glob(public_path.join("sitemap*.xml")).sort
+      storage.list(prefix: "sitemap", suffix: ".xml")
+    end
+
+    def sitemap_details(files)
+      files.map do |filename|
+        {
+          filename: filename.to_s,
+          location: sitemap_location(filename),
+          link_count: sitemap_link_count(filename)
+        }
+      end
+    end
+
+    def sitemap_location(filename)
+      return storage.public_url(filename) if storage.respond_to?(:public_url)
+
+      filename.to_s
+    end
+
+    def sitemap_link_count(filename)
+      document = Nokogiri::XML(storage.read(filename.to_s))
+      document.remove_namespaces!
+      document.xpath("//loc").count
     end
   end
 end

data/lib/indexmap/validator.rb

@@ -7,20 +7,17 @@ require "uri"
 
 module Indexmap
   class Validator
-    def initialize(configuration: Indexmap.configuration, path: nil)
+    def initialize(configuration: Indexmap.configuration, filename: nil)
       @configuration = configuration
-      @path = path
+      @filename = filename
     end
 
     def validate!
-      sitemap_path = path || Indexmap::Path.existing_public_path(
-        public_path: configuration.public_path,
-        index_filename: configuration.index_filename
-      )
-      raise ValidationError, "Missing sitemap file: #{sitemap_path}" unless File.exist?(sitemap_path)
-
-      validate_sitemap_file!(sitemap_path)
-      entries = Parser.new(path: sitemap_path).entries
+      sitemap_filename = filename || configuration.index_filename
+      raise ValidationError, "Missing sitemap file: #{sitemap_filename}" unless storage.exist?(sitemap_filename)
+
+      validate_sitemap_file!(sitemap_filename)
+      entries = Parser.new(source: sitemap_filename, storage: storage, index_filename: configuration.index_filename).entries
       validate_presence!(entries)
       validate_duplicates!(entries)
       validate_parameterized_urls!(entries)
@@ -33,39 +30,43 @@ module Indexmap
 
     private
 
-    attr_reader :configuration, :path
+    attr_reader :configuration, :filename
+
+    def storage
+      configuration.storage
+    end
 
-    def validate_sitemap_file!(sitemap_path)
-      document = read_xml_document(sitemap_path)
+    def validate_sitemap_file!(sitemap_filename)
+      document = read_xml_document(sitemap_filename)
       root_name = document.root&.name
 
       case root_name
       when "urlset"
-        validate_urlset_document!(document, sitemap_path)
+        validate_urlset_document!(document, sitemap_filename)
       when "sitemapindex"
-        validate_sitemap_index_document!(document, sitemap_path)
+        validate_sitemap_index_document!(document, sitemap_filename)
       else
-        raise ValidationError, "Invalid sitemap root element in #{sitemap_path}: #{root_name || "none"}"
+        raise ValidationError, "Invalid sitemap root element in #{sitemap_filename}: #{root_name || "none"}"
       end
     end
 
-    def read_xml_document(file_path)
-      document = Nokogiri::XML(File.read(file_path, encoding: "UTF-8")) { |config| config.strict }
+    def read_xml_document(filename)
+      document = Nokogiri::XML(storage.read(filename)) { |config| config.strict }
       document.remove_namespaces!
       document
     rescue Nokogiri::XML::SyntaxError => error
-      raise ValidationError, "Invalid sitemap XML in #{file_path}: #{error.message.lines.first.strip}"
+      raise ValidationError, "Invalid sitemap XML in #{filename}: #{error.message.lines.first.strip}"
     end
 
-    def validate_urlset_document!(document, sitemap_path)
+    def validate_urlset_document!(document, sitemap_filename)
       return if document.xpath("/urlset/url/loc").any?
 
-      raise ValidationError, "Sitemap has no URLs: #{sitemap_path}"
+      raise ValidationError, "Sitemap has no URLs: #{sitemap_filename}"
     end
 
-    def validate_sitemap_index_document!(document, sitemap_path)
+    def validate_sitemap_index_document!(document, sitemap_filename)
       child_locations = document.xpath("/sitemapindex/sitemap/loc").map { |node| node.text.to_s.strip }.reject(&:empty?)
-      raise ValidationError, "Sitemap index has no child sitemap URLs: #{sitemap_path}" if child_locations.empty?
+      raise ValidationError, "Sitemap index has no child sitemap URLs: #{sitemap_filename}" if child_locations.empty?
 
       duplicate_children = child_locations.group_by(&:itself).select { |_loc, values| values.size > 1 }.keys
       unless duplicate_children.empty?
@@ -73,19 +74,30 @@ module Indexmap
       end
 
       child_locations.each do |location|
-        child_path = local_child_path(sitemap_path, location)
-        raise ValidationError, "Missing child sitemap file: #{child_path}" unless File.exist?(child_path)
+        child_filename = local_child_filename(location, parent_filename: sitemap_filename)
+        raise ValidationError, "Missing child sitemap file: #{child_filename}" unless storage.exist?(child_filename)
 
-        validate_sitemap_file!(child_path)
+        validate_sitemap_file!(child_filename)
       end
     end
 
-    def local_child_path(sitemap_path, location)
+    def local_child_filename(location, parent_filename:)
       uri = URI.parse(location)
-      filename = (uri.absolute? || location.start_with?("/")) ? File.basename(uri.path) : location
-      File.expand_path(filename, File.dirname(sitemap_path))
+      if uri.absolute? || location.start_with?("/")
+        normalize_local_filename(uri.path)
+      else
+        parent_directory = File.dirname(parent_filename)
+        normalize_local_filename((parent_directory == ".") ? location : File.join(parent_directory, location))
+      end
     rescue URI::InvalidURIError
-      File.expand_path(location, File.dirname(sitemap_path))
+      normalize_local_filename(location)
+    end
+
+    def normalize_local_filename(filename)
+      normalized = Pathname(filename.to_s).cleanpath.to_s.sub(%r{\A/+}, "")
+      return if normalized.empty? || normalized == ".." || normalized.start_with?("../")
+
+      normalized
     end
 
     def validate_presence!(entries)

data/lib/indexmap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Indexmap
-  VERSION = "0.6.0"
+  VERSION = "0.7.1"
 end