indexmap 0.8.2 → 0.9.0

data/lib/indexmap/storage/filesystem.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "uri"
+require "tmpdir"
+require "active_support/core_ext/file/atomic"
 
 module Indexmap
   module Storage
@@ -8,16 +10,21 @@ module Indexmap
       DEFAULT_CONTENT_TYPE = "application/xml"
 
       def initialize(path:, public_url: nil)
-        @path = Pathname(path)
+        @path = Pathname(path).expand_path
+        @path.mkpath
+        @path = @path.realpath
         @public_url_base = public_url
       end
 
       def write(filename, body, content_type: DEFAULT_CONTENT_TYPE)
+        if Publication.immutable?(filename) && exist?(filename)
+          raise ConfigurationError, "Immutable sitemap already exists: #{filename}"
+        end
         target = path_for(filename)
         target.dirname.mkpath
-        target.write(body.to_s)
+        ::File.atomic_write(target) { |file| file.write(body.to_s) }
 
-        File.new(
+        Artifact.new(
           filename: normalize_filename(filename),
           body: body.to_s,
           content_type: content_type || DEFAULT_CONTENT_TYPE
@@ -33,13 +40,32 @@ module Indexmap
       end
 
       def list(prefix: nil, suffix: nil)
-        path.glob("**/*").select(&:file?).filter_map do |file|
-          filename = file.relative_path_from(path).to_s
-          next if prefix && !filename.start_with?(prefix)
-          next if suffix && !filename.end_with?(suffix)
-
-          filename
-        end.sort
+        prefix = prefix.to_s
+        unless prefix.empty? || normalize_filename(prefix) == prefix.delete_suffix("/")
+          raise ArgumentError, "Storage prefix must be a normalized relative path: #{prefix.inspect}"
+        end
+        directory, _, fragment = prefix.rpartition("/")
+        ancestor = path
+        directory.split("/").each do |segment|
+          ancestor = ancestor.join(segment)
+          return [] if segment.start_with?(".") || ancestor.symlink?
+        end
+        base = directory.empty? ? path : path_for(directory)
+        return [] unless base.directory?
+
+        pending = base.children.select { |entry| entry.basename.to_s.start_with?(fragment) }
+        filenames = []
+        until pending.empty?
+          entry = pending.pop
+          next if entry.symlink? || entry.basename.to_s.start_with?(".")
+
+          if entry.directory?
+            pending.concat(entry.children)
+          elsif (!suffix || entry.to_s.end_with?(suffix)) && entry.file?
+            filenames << entry.relative_path_from(path).to_s
+          end
+        end
+        filenames.sort
       end
 
       def delete(filename)
@@ -52,6 +78,73 @@ module Indexmap
         URI.join("#{public_url_base.to_s.delete_suffix("/")}/", normalize_filename(filename)).to_s
       end
 
+      def open(filename, &block)
+        path_for(filename).open("rb", &block)
+      end
+
+      def write_io(filename, io, content_type: DEFAULT_CONTENT_TYPE)
+        if Publication.immutable?(filename) && exist?(filename)
+          raise ConfigurationError, "Immutable sitemap already exists: #{filename}"
+        end
+        target = path_for(filename)
+        target.dirname.mkpath
+        ::File.atomic_write(target) { |file| IO.copy_stream(io, file) }
+      end
+
+      def metadata(filename)
+        stat = path_for(filename).stat
+        {etag: "#{stat.ino}-#{stat.size}-#{stat.mtime.to_r}", bytes: stat.size, last_modified: stat.mtime.utc}
+      end
+
+      def revision(root)
+        exist?(root) ? metadata(root).fetch(:etag) : nil
+      end
+
+      def publish(source, files:, root:, expected:)
+        check_revision!(root, expected)
+        publication = Publication.new(root: root)
+        publication.validate_files!(files)
+        with_staging(root, expected: expected) do |staged|
+          files.each do |filename|
+            source.open(filename) { |io| staged.write_io(filename, io) }
+          end
+          archive = if expected && (version = publication.current_generation(self))
+            publication.archive(version).tap do |filename|
+              self.open(root) { |io| staged.write_io(filename, io) }
+            end
+          end
+          with_lock(root) do
+            check_revision!(root, expected)
+            children = files.reject { |filename| filename == root }
+            children.each do |filename|
+              raise ConfigurationError, "Immutable sitemap already exists: #{filename}" if exist?(filename)
+            end
+            moved = []
+            root_revision = staged.revision(root)
+            begin
+              children.each do |filename|
+                move_staged(staged, filename)
+                moved << filename
+              end
+              move_staged(staged, archive, retired_at: Time.now.utc) if archive
+              move_staged(staged, root)
+            ensure
+              # A root rename can succeed immediately before an interrupt.
+              moved.each { |filename| delete(filename) } unless revision(root) == root_revision
+            end
+          end
+        end
+      rescue Errno::ENOENT
+        check_revision!(root, expected)
+        raise
+      end
+
+      def prune(root:, before:)
+        with_lock(root) do
+          Publication.new(root: root).prune(self, before: before) + prune_staging(root, before: before)
+        end
+      end
+
       def inspect
         "#<#{self.class.name} path=#{path}>"
       end
@@ -60,8 +153,86 @@ module Indexmap
 
       attr_reader :path, :public_url_base
 
+      def check_revision!(root, expected)
+        raise SupersededPublication, "Sitemap changed while building: #{root}" unless revision(root) == expected
+      end
+
+      def staging_path(root)
+        path_for(".indexmap-builds/#{Digest::SHA256.hexdigest(root)}")
+      end
+
+      def with_staging(root, expected:)
+        directory = nil
+        lock = nil
+        with_lock(root) do
+          check_revision!(root, expected)
+          base = staging_path(root)
+          base.mkpath
+          directory = Dir.mktmpdir("build-", base)
+          lock = ::File.open(::File.join(directory, ".lock"), "w")
+          lock.flock(::File::LOCK_EX)
+        end
+        yield Filesystem.new(path: directory)
+      ensure
+        begin
+          FileUtils.remove_entry(directory) if directory
+        ensure
+          lock&.close
+        end
+      end
+
+      def move_staged(staged, filename, retired_at: nil)
+        target = path_for(filename)
+        target.dirname.mkpath
+        staged.open(filename) do |io|
+          ::File.utime(retired_at, retired_at, io.path) if retired_at
+          ::File.atomic_write(target) do |temporary|
+            temporary.close
+            ::File.rename(io.path, temporary.path)
+          end
+        end
+      end
+
+      def prune_staging(root, before:)
+        base = staging_path(root)
+        return [] unless base.directory?
+
+        base.children.filter_map do |directory|
+          next if directory.symlink? || !directory.directory? || directory.mtime >= before
+
+          begin
+            directory.join(".lock").open("a") do |lock|
+              next unless lock.flock(::File::LOCK_EX | ::File::LOCK_NB)
+
+              FileUtils.remove_entry_secure(directory.to_s)
+              directory.relative_path_from(path).to_s
+            end
+          rescue Errno::ENOENT
+            # A completed build can remove its staging directory during pruning.
+            nil
+          end
+        end
+      end
+
+      def with_lock(root)
+        lock = path_for(".indexmap-#{Digest::SHA256.hexdigest(root)}.lock")
+        lock.open("a") do |file|
+          file.flock(::File::LOCK_EX)
+          yield
+        ensure
+          file.flock(::File::LOCK_UN)
+        end
+      end
+
       def path_for(filename)
-        path.join(normalize_filename(filename))
+        target = path.join(normalize_filename(filename))
+        ancestor = target
+        ancestor = ancestor.parent until ancestor.exist? || ancestor.symlink?
+        resolved = ancestor.realpath
+        unless resolved == path || resolved.to_s.start_with?("#{path}/")
+          raise ArgumentError, "Storage filename escapes its root: #{filename.inspect}"
+        end
+        target
       end
 
       def normalize_filename(filename)

data/lib/indexmap/storage/memory.rb

@@ -17,7 +17,7 @@ module Indexmap
 
       def write(filename, body, content_type: DEFAULT_CONTENT_TYPE)
         normalized = normalize_filename(filename)
-        files[normalized] = File.new(
+        files[normalized] = Artifact.new(
           filename: normalized,
           body: body.to_s,
           content_type: content_type || DEFAULT_CONTENT_TYPE

data/lib/indexmap/task_runner.rb

@@ -4,24 +4,29 @@ require "nokogiri"
 
 module Indexmap
   class TaskRunner
-    def initialize(configuration: Indexmap.configuration)
+    def initialize(configuration: Indexmap.configuration, storage: nil)
       @configuration = configuration
+      @storage = storage
     end
 
     def create
-      written_files = Indexmap.create(configuration: configuration)
+      creator = Creator.new(output: default_output, storage: storage)
+      written_files = creator.create
       index_now_key_filename = write_index_now_key if configuration.index_now.write_key_file?
       configuration.run_after_create_callbacks
 
       {
         files: written_files.map(&:to_s),
         written_files: written_files,
-        sitemaps: sitemap_details(written_files),
+        sitemaps: creator.sitemaps,
         index_now_key_filename: index_now_key_filename
       }
     end
 
     def format
+      if configuration.output_names.any? { |name| configuration.output_for(name).atomic }
+        raise ConfigurationError, "Format immutable sitemaps by regenerating them"
+      end
       sitemap_files.each do |filename|
         content = storage.read(filename)
         document = Nokogiri::XML(
@@ -39,19 +44,29 @@ module Indexmap
     end
 
     def validate
-      Validator.new(configuration: configuration).validate!
+      Validator.new(configuration: configuration, storage: storage).validate!(collect: false)
       sitemap_files
     end
 
+    def prune(retention: 48 * 3600)
+      raise ConfigurationError, "Retention must be at least one hour" unless retention.is_a?(Numeric) && retention >= 3600
+      raise ConfigurationError, "Storage does not support pruning" unless storage.respond_to?(:prune)
+
+      configuration.output_names.flat_map do |name|
+        output = configuration.output_for(name)
+        output.atomic ? storage.prune(root: output.index_filename, before: Time.now.utc - retention) : []
+      end
+    end
+
     def write_index_now_key(generate_if_missing: false)
-      pinger = Pinger::IndexNow.new(configuration: configuration)
+      pinger = Pinger::IndexNow.new(configuration: configuration, storage: storage)
       return pinger.ensure_key_file if generate_if_missing
 
       pinger.write_key_file
     end
 
     def storage
-      configuration.storage
+      @storage ||= configuration.storage
     end
 
     private
@@ -65,27 +80,5 @@ module Indexmap
     def sitemap_files
       storage.list(prefix: "sitemap", suffix: ".xml")
     end
-
-    def sitemap_details(files)
-      files.map do |filename|
-        {
-          filename: filename.to_s,
-          location: sitemap_location(filename),
-          link_count: sitemap_link_count(filename)
-        }
-      end
-    end
-
-    def sitemap_location(filename)
-      return storage.public_url(filename) if storage.respond_to?(:public_url)
-
-      filename.to_s
-    end
-
-    def sitemap_link_count(filename)
-      document = Nokogiri::XML(storage.read(filename.to_s))
-      document.remove_namespaces!
-      document.xpath("//loc").count
-    end
   end
 end

data/lib/indexmap/validator/duplicates.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Indexmap
+  class Validator
+    class Duplicates
+      def initialize(directory:)
+        @directory = directory
+        @buffer = []
+        @runs = []
+        @sequence = 0
+      end
+
+      def add(location)
+        @buffer << location
+        flush if @buffer.size == 10_000
+      end
+
+      def validate!
+        previous = nil
+        each_sorted_location do |line|
+          raise ValidationError, "Duplicate sitemap URLs detected: #{line.chomp}" if line == previous
+
+          previous = line
+        end
+      end
+
+      def unique_count
+        count = 0
+        previous = nil
+        each_sorted_location do |line|
+          count += 1 unless line == previous
+          previous = line
+        end
+        count
+      end
+
+      private
+
+      def each_sorted_location
+        flush unless @buffer.empty?
+        while @runs.size > 1
+          @runs = @runs.each_slice(2).map do |left, right|
+            next left unless right
+
+            target = next_path
+            File.open(left) do |a|
+              File.open(right) do |b|
+                File.open(target, "wb") do |output|
+                  x = a.gets
+                  y = b.gets
+                  while x || y
+                    if y.nil? || (x && x <= y)
+                      output.write(x)
+                      x = a.gets
+                    else
+                      output.write(y)
+                      y = b.gets
+                    end
+                  end
+                end
+              end
+            end
+            File.unlink(left, right)
+            target
+          end
+        end
+        File.foreach(@runs.first) { |line| yield line } if @runs.first
+      end
+
+      def flush
+        path = next_path
+        File.open(path, "wb") { |file| @buffer.sort.each { |location| file.puts(location) } }
+        @runs << path
+        @buffer.clear
+      end
+
+      def next_path
+        @sequence += 1
+        File.join(@directory, "urls-#{@sequence}")
+      end
+    end
+  end
+end