indexmap 0.5.0 → 0.7.0

data/lib/indexmap/pinger/index_now.rb

@@ -42,14 +42,15 @@ module Indexmap
         summarize_results(results)
       end
 
-      def write_key_file(key: index_now_configuration.key, path: nil)
+      def write_key_file(key: index_now_configuration.key, filename: nil)
         key = normalized_configured_key(key)
         return if key.empty?
 
-        path ||= index_now_configuration.key_path(public_path: configuration.public_path, key: key)
-        FileUtils.mkdir_p(path.dirname)
-        File.write(path, key)
-        path
+        filename ||= index_now_configuration.key_filename(key: key)
+        return filename if valid_key_file?(filename)
+
+        storage.write(filename, key, content_type: "text/plain")
+        filename
       end
 
       def ensure_key_file
@@ -60,7 +61,7 @@ module Indexmap
         return existing_path if existing_path
 
         key = generated_key
-        write_key_file(key: key, path: configuration.public_path.join("#{key}.txt"))
+        write_key_file(key: key, filename: "#{key}.txt")
       end
 
       private
@@ -75,7 +76,7 @@ module Indexmap
         files = super
         return files if files.one?
 
-        child_files = files.reject { |file| File.basename(file) == configuration.index_filename }
+        child_files = files.reject { |file| file == configuration.index_filename }
         child_files.empty? ? files : child_files
       end
 
@@ -95,7 +96,7 @@ module Indexmap
 
       def current_entries
         sitemap_files.each_with_object({}) do |sitemap_file, entries|
-          Parser.new(path: sitemap_file).entries.each do |entry|
+          Parser.new(source: sitemap_file, storage: storage).entries.each do |entry|
             next if entry.loc.to_s.strip.empty?
 
             entries[entry.loc] = entry
@@ -177,36 +178,32 @@ module Indexmap
         configured_key = normalized_configured_key(index_now_configuration.key)
         return configured_key unless configured_key.empty?
 
-        existing_key_file&.read
+        storage.read(existing_key_file) if existing_key_file
       end
 
       def existing_key_file
-        configured_path = index_now_configuration.key_path(public_path: configuration.public_path)
-        return configured_path if valid_key_file?(configured_path)
+        configured_filename = index_now_configuration.key_filename
+        return configured_filename if valid_key_file?(configured_filename)
 
-        configuration.public_path.glob("*.txt").sort.find { |file| valid_key_file?(file) }
+        storage.list(suffix: ".txt").find { |filename| valid_key_file?(filename) }
       end
 
       def key_location(api_key:)
-        path = index_now_configuration.key_path(public_path: configuration.public_path, key: api_key) || existing_key_file
-        return unless path
-
-        public_path = configuration.public_path.expand_path
-        key_path = path.expand_path
-        relative_path = key_path.relative_path_from(public_path)
+        filename = index_now_configuration.key_filename(key: api_key) || existing_key_file
+        return unless filename
 
-        URI.join("#{host}/", relative_path.to_s).to_s
+        storage.public_url(filename)
       rescue ArgumentError
         nil
       end
 
-      def valid_key_file?(path)
-        return false unless path&.file?
+      def valid_key_file?(filename)
+        return false unless filename && storage.exist?(filename)
 
-        filename = path.basename(".txt").to_s
-        return false unless filename.match?(KEY_FORMAT)
+        key = File.basename(filename, ".txt").to_s
+        return false unless key.match?(KEY_FORMAT)
 
-        path.read == filename
+        storage.read(filename) == key
       end
 
       def generated_key

data/lib/indexmap/storage/active_storage.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "uri"
+
+module Indexmap
+  module Storage
+    class ActiveStorage
+      DEFAULT_CONTENT_TYPE = "application/xml"
+
+      def initialize(model:, public_url:, filename_column: :filename, attachment: :file, content_type: DEFAULT_CONTENT_TYPE)
+        @model = model
+        @public_url_base = public_url
+        @filename_column = filename_column
+        @attachment = attachment
+        @default_content_type = content_type
+      end
+
+      def write(filename, body, content_type: nil)
+        attachment_content_type = content_type || default_content_type
+        record = find_or_initialize(filename)
+        record.save! unless record.persisted?
+        record.public_send(attachment).attach(
+          io: StringIO.new(body.to_s),
+          filename: filename,
+          content_type: attachment_content_type
+        )
+
+        File.new(filename: filename, body: body.to_s, content_type: attachment_content_type)
+      end
+
+      def read(filename)
+        record = find_record(filename)
+        return unless attached?(record)
+
+        record.public_send(attachment).download
+      end
+
+      def exist?(filename)
+        attached?(find_record(filename))
+      end
+
+      def list(prefix: nil, suffix: nil)
+        relation = relation_for(prefix: prefix, suffix: suffix)
+
+        relation.filter_map do |record|
+          filename = record.public_send(filename_column).to_s
+          next if prefix && !filename.start_with?(prefix)
+          next if suffix && !filename.end_with?(suffix)
+          next unless attached?(record)
+
+          filename
+        end.sort
+      end
+
+      def delete(filename)
+        record = find_record(filename)
+        record&.public_send(attachment)&.purge
+      end
+
+      def public_url(filename)
+        URI.join("#{public_url_base}/", filename).to_s
+      end
+
+      def inspect
+        "#<#{self.class.name} model=#{model}>"
+      end
+
+      private
+
+      attr_reader :model, :filename_column, :attachment, :default_content_type
+
+      def find_or_initialize(filename)
+        model.find_or_initialize_by(filename_column => filename)
+      end
+
+      def find_record(filename)
+        model.find_by(filename_column => filename)
+      end
+
+      def relation_for(prefix:, suffix:)
+        if !model.respond_to?(:where) || (!prefix && !suffix)
+          return model.all
+        end
+
+        column = model.connection.quote_column_name(filename_column)
+        if prefix && suffix
+          model.where("#{column} LIKE ?", "#{prefix}%#{suffix}")
+        elsif prefix
+          model.where("#{column} LIKE ?", "#{prefix}%")
+        else
+          model.where("#{column} LIKE ?", "%#{suffix}")
+        end
+      end
+
+      def attached?(record)
+        record&.public_send(attachment)&.attached?
+      end
+
+      def public_url_base
+        @public_url_base.to_s.delete_suffix("/")
+      end
+    end
+  end
+end

data/lib/indexmap/storage/file.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Indexmap
+  module Storage
+    File = Struct.new(:filename, :body, :content_type, keyword_init: true) do
+      def basename
+        ::File.basename(filename)
+      end
+    end
+  end
+end

data/lib/indexmap/storage/filesystem.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Indexmap
+  module Storage
+    class Filesystem
+      DEFAULT_CONTENT_TYPE = "application/xml"
+
+      def initialize(path:, public_url: nil)
+        @path = Pathname(path)
+        @public_url_base = public_url
+      end
+
+      def write(filename, body, content_type: DEFAULT_CONTENT_TYPE)
+        target = path_for(filename)
+        target.dirname.mkpath
+        target.write(body.to_s)
+
+        File.new(
+          filename: normalize_filename(filename),
+          body: body.to_s,
+          content_type: content_type || DEFAULT_CONTENT_TYPE
+        )
+      end
+
+      def read(filename)
+        path_for(filename).read(encoding: "UTF-8")
+      end
+
+      def exist?(filename)
+        path_for(filename).file?
+      end
+
+      def list(prefix: nil, suffix: nil)
+        path.glob("**/*").select(&:file?).filter_map do |file|
+          filename = file.relative_path_from(path).to_s
+          next if prefix && !filename.start_with?(prefix)
+          next if suffix && !filename.end_with?(suffix)
+
+          filename
+        end.sort
+      end
+
+      def delete(filename)
+        path_for(filename).delete if exist?(filename)
+      end
+
+      def public_url(filename)
+        return normalize_filename(filename) if public_url_base.to_s.strip.empty?
+
+        URI.join("#{public_url_base.to_s.delete_suffix("/")}/", normalize_filename(filename)).to_s
+      end
+
+      def inspect
+        "#<#{self.class.name} path=#{path}>"
+      end
+
+      private
+
+      attr_reader :path, :public_url_base
+
+      def path_for(filename)
+        path.join(normalize_filename(filename))
+      end
+
+      def normalize_filename(filename)
+        normalized = Pathname(filename.to_s).cleanpath
+        if normalized.absolute? || normalized.to_s.start_with?("../") || normalized.to_s == ".."
+          raise ArgumentError, "Storage filename must be relative: #{filename.inspect}"
+        end
+
+        normalized.to_s
+      end
+    end
+  end
+end

data/lib/indexmap/storage/memory.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Indexmap
+  module Storage
+    class Memory
+      DEFAULT_CONTENT_TYPE = "application/xml"
+
+      def initialize(files = [], public_url: nil)
+        @files = {}
+        @public_url_base = public_url
+        Array(files).each do |file|
+          write(file.filename, file.body, content_type: file.content_type)
+        end
+      end
+
+      def write(filename, body, content_type: DEFAULT_CONTENT_TYPE)
+        normalized = normalize_filename(filename)
+        files[normalized] = File.new(
+          filename: normalized,
+          body: body.to_s,
+          content_type: content_type || DEFAULT_CONTENT_TYPE
+        )
+      end
+
+      def read(filename)
+        files.fetch(normalize_filename(filename)).body
+      end
+
+      def exist?(filename)
+        files.key?(normalize_filename(filename))
+      end
+
+      def list(prefix: nil, suffix: nil)
+        files.keys.select do |filename|
+          (prefix.nil? || filename.start_with?(prefix)) &&
+            (suffix.nil? || filename.end_with?(suffix))
+        end.sort
+      end
+
+      def delete(filename)
+        files.delete(normalize_filename(filename))
+      end
+
+      def public_url(filename)
+        return normalize_filename(filename) if public_url_base.to_s.strip.empty?
+
+        URI.join("#{public_url_base.to_s.delete_suffix("/")}/", normalize_filename(filename)).to_s
+      end
+
+      private
+
+      attr_reader :files, :public_url_base
+
+      def normalize_filename(filename)
+        filename.to_s
+      end
+    end
+  end
+end

data/lib/indexmap/task_runner.rb

@@ -9,14 +9,16 @@ module Indexmap
     end
 
     def create
-      remove_existing_sitemap_files
-      configuration.writer.write
-      {files: sitemap_files, index_now_key_path: write_index_now_key}
+      written_files = Indexmap.create(configuration: configuration)
+      index_now_key_filename = write_index_now_key if configuration.index_now.write_key_file?
+      configuration.run_after_create_callbacks
+
+      {files: written_files.map(&:to_s), written_files: written_files, index_now_key_filename: index_now_key_filename}
     end
 
     def format
-      sitemap_files.each do |file_path|
-        content = File.read(file_path)
+      sitemap_files.each do |filename|
+        content = storage.read(filename)
         document = Nokogiri::XML(
           content,
           nil,
@@ -25,7 +27,7 @@ module Indexmap
         )
         save_options = Nokogiri::XML::Node::SaveOptions::FORMAT | Nokogiri::XML::Node::SaveOptions::AS_XML
 
-        File.write(file_path, document.to_xml(indent: 2, save_with: save_options))
+        storage.write(filename, document.to_xml(indent: 2, save_with: save_options), content_type: "application/xml")
       end
 
       sitemap_files
@@ -43,22 +45,20 @@ module Indexmap
       pinger.write_key_file
     end
 
-    def public_path
-      configuration.public_path
+    def storage
+      configuration.storage
     end
 
     private
 
     attr_reader :configuration
 
-    def remove_existing_sitemap_files
-      Dir.glob(configuration.public_path.join("sitemap*.xml*")).each do |file_path|
-        File.delete(file_path)
-      end
+    def default_output
+      configuration.output_for(:default)
     end
 
     def sitemap_files
-      Dir.glob(configuration.public_path.join("sitemap*.xml")).sort
+      storage.list(prefix: "sitemap", suffix: ".xml")
     end
   end
 end

data/lib/indexmap/validator.rb

@@ -7,20 +7,17 @@ require "uri"
 
 module Indexmap
   class Validator
-    def initialize(configuration: Indexmap.configuration, path: nil)
+    def initialize(configuration: Indexmap.configuration, filename: nil)
       @configuration = configuration
-      @path = path
+      @filename = filename
     end
 
     def validate!
-      sitemap_path = path || Indexmap::Path.existing_public_path(
-        public_path: configuration.public_path,
-        index_filename: configuration.index_filename
-      )
-      raise ValidationError, "Missing sitemap file: #{sitemap_path}" unless File.exist?(sitemap_path)
-
-      validate_sitemap_file!(sitemap_path)
-      entries = Parser.new(path: sitemap_path).entries
+      sitemap_filename = filename || configuration.index_filename
+      raise ValidationError, "Missing sitemap file: #{sitemap_filename}" unless storage.exist?(sitemap_filename)
+
+      validate_sitemap_file!(sitemap_filename)
+      entries = Parser.new(source: sitemap_filename, storage: storage, index_filename: configuration.index_filename).entries
       validate_presence!(entries)
       validate_duplicates!(entries)
       validate_parameterized_urls!(entries)
@@ -33,39 +30,43 @@ module Indexmap
 
     private
 
-    attr_reader :configuration, :path
+    attr_reader :configuration, :filename
+
+    def storage
+      configuration.storage
+    end
 
-    def validate_sitemap_file!(sitemap_path)
-      document = read_xml_document(sitemap_path)
+    def validate_sitemap_file!(sitemap_filename)
+      document = read_xml_document(sitemap_filename)
       root_name = document.root&.name
 
       case root_name
       when "urlset"
-        validate_urlset_document!(document, sitemap_path)
+        validate_urlset_document!(document, sitemap_filename)
       when "sitemapindex"
-        validate_sitemap_index_document!(document, sitemap_path)
+        validate_sitemap_index_document!(document, sitemap_filename)
       else
-        raise ValidationError, "Invalid sitemap root element in #{sitemap_path}: #{root_name || "none"}"
+        raise ValidationError, "Invalid sitemap root element in #{sitemap_filename}: #{root_name || "none"}"
       end
     end
 
-    def read_xml_document(file_path)
-      document = Nokogiri::XML(File.read(file_path, encoding: "UTF-8")) { |config| config.strict }
+    def read_xml_document(filename)
+      document = Nokogiri::XML(storage.read(filename)) { |config| config.strict }
       document.remove_namespaces!
       document
     rescue Nokogiri::XML::SyntaxError => error
-      raise ValidationError, "Invalid sitemap XML in #{file_path}: #{error.message.lines.first.strip}"
+      raise ValidationError, "Invalid sitemap XML in #{filename}: #{error.message.lines.first.strip}"
     end
 
-    def validate_urlset_document!(document, sitemap_path)
+    def validate_urlset_document!(document, sitemap_filename)
       return if document.xpath("/urlset/url/loc").any?
 
-      raise ValidationError, "Sitemap has no URLs: #{sitemap_path}"
+      raise ValidationError, "Sitemap has no URLs: #{sitemap_filename}"
     end
 
-    def validate_sitemap_index_document!(document, sitemap_path)
+    def validate_sitemap_index_document!(document, sitemap_filename)
       child_locations = document.xpath("/sitemapindex/sitemap/loc").map { |node| node.text.to_s.strip }.reject(&:empty?)
-      raise ValidationError, "Sitemap index has no child sitemap URLs: #{sitemap_path}" if child_locations.empty?
+      raise ValidationError, "Sitemap index has no child sitemap URLs: #{sitemap_filename}" if child_locations.empty?
 
       duplicate_children = child_locations.group_by(&:itself).select { |_loc, values| values.size > 1 }.keys
       unless duplicate_children.empty?
@@ -73,19 +74,30 @@ module Indexmap
       end
 
       child_locations.each do |location|
-        child_path = local_child_path(sitemap_path, location)
-        raise ValidationError, "Missing child sitemap file: #{child_path}" unless File.exist?(child_path)
+        child_filename = local_child_filename(location, parent_filename: sitemap_filename)
+        raise ValidationError, "Missing child sitemap file: #{child_filename}" unless storage.exist?(child_filename)
 
-        validate_sitemap_file!(child_path)
+        validate_sitemap_file!(child_filename)
       end
     end
 
-    def local_child_path(sitemap_path, location)
+    def local_child_filename(location, parent_filename:)
       uri = URI.parse(location)
-      filename = (uri.absolute? || location.start_with?("/")) ? File.basename(uri.path) : location
-      File.expand_path(filename, File.dirname(sitemap_path))
+      if uri.absolute? || location.start_with?("/")
+        normalize_local_filename(uri.path)
+      else
+        parent_directory = File.dirname(parent_filename)
+        normalize_local_filename((parent_directory == ".") ? location : File.join(parent_directory, location))
+      end
     rescue URI::InvalidURIError
-      File.expand_path(location, File.dirname(sitemap_path))
+      normalize_local_filename(location)
+    end
+
+    def normalize_local_filename(filename)
+      normalized = Pathname(filename.to_s).cleanpath.to_s.sub(%r{\A/+}, "")
+      return if normalized.empty? || normalized == ".." || normalized.start_with?("../")
+
+      normalized
     end
 
     def validate_presence!(entries)

data/lib/indexmap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Indexmap
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

data/lib/indexmap/writer.rb

@@ -4,30 +4,27 @@ module Indexmap
   class Writer
     VALID_FORMATS = %i[index single_file].freeze
 
-    def initialize(public_path:, base_url:, sections: nil, entries: nil, index_filename: "sitemap.xml", format: :index)
+    def initialize(base_url:, sections: nil, entries: nil, index_filename: "sitemap.xml", format: :index)
       @entries = normalize_entries(entries)
       @format = normalize_format(format)
       @sections = normalize_sections(sections)
-      @public_path = Pathname(public_path)
       @base_url = base_url
       @index_filename = index_filename
     end
 
     def write
-      FileUtils.mkdir_p(public_path)
+      return [write_file(index_filename, urlset_xml(entries))] if single_file?
 
-      return public_path.join(index_filename).write(urlset_xml(entries)) if single_file?
-
-      sections.each do |section|
-        public_path.join(section.filename).write(urlset_xml(section.entries))
+      paths = sections.map do |section|
+        write_file(section.filename, urlset_xml(section.entries))
       end
 
-      public_path.join(index_filename).write(index_xml(sections))
+      paths + [write_file(index_filename, index_xml(sections))]
     end
 
     private
 
-    attr_reader :base_url, :entries, :format, :index_filename, :public_path, :sections
+    attr_reader :base_url, :entries, :format, :index_filename, :sections
 
     def normalize_entries(raw_entries)
       Array(raw_entries).map { |entry| normalize_entry(entry) }
@@ -55,6 +52,10 @@ module Indexmap
       format == :single_file
     end
 
+    def write_file(filename, body)
+      Storage::File.new(filename: filename, body: body, content_type: "application/xml")
+    end
+
     def urlset_xml(entries)
       lines = [
         %(<?xml version="1.0" encoding="UTF-8"?>),

data/lib/indexmap.rb

@@ -3,15 +3,19 @@
 require "cgi"
 require "date"
 require "fileutils"
-require "pathname"
 require "time"
 
 require_relative "indexmap/version"
 require_relative "indexmap/google_configuration"
 require_relative "indexmap/index_now_configuration"
+require_relative "indexmap/storage/file"
+require_relative "indexmap/storage/memory"
+require_relative "indexmap/storage/filesystem"
+require_relative "indexmap/storage/active_storage"
 require_relative "indexmap/configuration"
+require_relative "indexmap/creator"
 require_relative "indexmap/entry"
-require_relative "indexmap/path"
+require_relative "indexmap/output"
 require_relative "indexmap/parser"
 require_relative "indexmap/pinger/base"
 require_relative "indexmap/pinger/google"
@@ -39,6 +43,10 @@ module Indexmap
     def reset!
       @configuration = Configuration.new
     end
+
+    def create(output_name = :default, configuration: self.configuration)
+      Creator.new(output: configuration.output_for(output_name)).create
+    end
   end
 end
 

data/lib/tasks/indexmap_tasks.rake

@@ -4,11 +4,9 @@ namespace :indexmap do
     task create: :environment do
       runner = Indexmap::TaskRunner.new
       create_result = runner.create
-      runner.format
-      validated_files = runner.validate
 
-      puts "Created, formatted, and validated #{file_count(validated_files)} in #{public_directory(runner)}."
-      puts "IndexNow key file: #{create_result[:index_now_key_path]}" if create_result[:index_now_key_path]
+      puts "Created, formatted, and validated #{file_count(create_result[:files])} in #{storage_description(runner)}."
+      puts "IndexNow key file: #{create_result[:index_now_key_filename]}" if create_result[:index_now_key_filename]
     end
 
     desc "Format sitemap files for better readability"
@@ -16,7 +14,7 @@ namespace :indexmap do
       runner = Indexmap::TaskRunner.new
       formatted_files = runner.format
 
-      puts "Formatted #{file_count(formatted_files)} in #{public_directory(runner)}."
+      puts "Formatted #{file_count(formatted_files)} in #{storage_description(runner)}."
     end
 
     desc "Validate sitemap shape and URL hygiene"
@@ -41,9 +39,9 @@ namespace :indexmap do
 
       case result[:status]
       when :submitted
-        puts "Submitted #{result[:sitemap_count]} sitemap #{(result[:sitemap_count] == 1) ? "file" : "files"} to Google Search Console."
+        puts "Submitted #{result[:sitemap_count]} sitemap #{(result[:sitemap_count] == 1) ? "file" : "files"} with #{result[:url_count]} URL#{"s" unless result[:url_count] == 1} to Google Search Console."
       when :partial
-        puts "Submitted #{result[:sitemap_count]} sitemap #{(result[:sitemap_count] == 1) ? "file" : "files"} to Google Search Console, with #{result[:failures].count} failure#{"s" unless result[:failures].count == 1}."
+        puts "Submitted #{result[:sitemap_count]} sitemap #{(result[:sitemap_count] == 1) ? "file" : "files"} with #{result[:url_count]} URL#{"s" unless result[:url_count] == 1} to Google Search Console, with #{result[:failures].count} failure#{"s" unless result[:failures].count == 1}."
         result[:failures].each { |failure| puts format_google_ping_failure(failure) }
       when :failed
         result[:failures].each { |failure| puts format_google_ping_failure(failure) }
@@ -89,8 +87,8 @@ namespace :indexmap do
     "#{count} sitemap #{(count == 1) ? "file" : "files"}"
   end
 
-  def public_directory(runner)
-    runner.public_path
+  def storage_description(runner)
+    runner.storage
   end
 
   def format_google_ping_failure(failure)