indexmap 0.8.2 → 0.9.0

data/lib/indexmap/validator.rb

@@ -4,85 +4,202 @@ require "nokogiri"
 require "date"
 require "time"
 require "uri"
+require "tmpdir"
+require "stringio"
+require "json"
 
 module Indexmap
   class Validator
-    def initialize(configuration: Indexmap.configuration, filename: nil)
+    MAX_ENTRIES = 50_000
+    MAX_BYTES = 52_428_800
+
+    attr_reader :entries
+
+    def initialize(configuration: Indexmap.configuration, filename: nil, storage: nil)
       @configuration = configuration
+      @storage = storage
       @filename = filename
     end
 
-    def validate!
+    def validate!(collect: true)
       sitemap_filename = filename || configuration.index_filename
       raise ValidationError, "Missing sitemap file: #{sitemap_filename}" unless storage.exist?(sitemap_filename)
 
-      validate_sitemap_file!(sitemap_filename)
-      entries = Parser.new(source: sitemap_filename, storage: storage, index_filename: configuration.index_filename).entries
-      validate_presence!(entries)
-      validate_duplicates!(entries)
-      validate_parameterized_urls!(entries)
-      validate_fragment_urls!(entries)
-      validate_absolute_http_urls!(entries)
-      validate_same_host_urls!(entries)
-      validate_lastmods!(entries)
+      @entries = []
+      count = 0
+      Dir.mktmpdir("indexmap-validation") do |directory|
+        duplicates = Duplicates.new(directory: directory)
+        validate_sitemap_files!(sitemap_filename) do |entry|
+          validate_parameterized_urls!([entry])
+          validate_fragment_urls!([entry])
+          validate_absolute_http_urls!([entry])
+          validate_same_host_urls!([entry])
+          validate_lastmods!([entry])
+          yield entry if block_given?
+          duplicates.add(entry.loc)
+          @entries << entry if collect
+          count += 1
+        end
+        duplicates.validate!
+      end
+      raise ValidationError, "Sitemap has no URLs" if count == 0
+
       true
     end
 
+    def each_entry
+      return enum_for(:each_entry) unless block_given?
+
+      Dir.mktmpdir("indexmap-entries") do |directory|
+        path = File.join(directory, "entries.jsonl")
+        File.open(path, "wb") do |file|
+          validate!(collect: false) { |entry| file.puts(JSON.generate(entry.to_h)) }
+        end
+        File.foreach(path) do |line|
+          yield Parser::Entry.new(**JSON.parse(line, symbolize_names: true))
+        end
+      end
+    end
+
     private
 
     attr_reader :configuration, :filename
 
     def storage
-      configuration.storage
+      @storage ||= configuration.storage
     end
 
-    def validate_sitemap_file!(sitemap_filename)
-      document = read_xml_document(sitemap_filename)
-      root_name = document.root&.name
-
-      case root_name
-      when "urlset"
-        validate_urlset_document!(document, sitemap_filename)
-      when "sitemapindex"
-        validate_sitemap_index_document!(document, sitemap_filename)
-      else
-        raise ValidationError, "Invalid sitemap root element in #{sitemap_filename}: #{root_name || "none"}"
+    def validate_sitemap_files!(sitemap_filename)
+      pending = [[sitemap_filename, false]]
+      active = Set.new
+      visited = Set.new
+      until pending.empty?
+        name, leaving = pending.pop
+        if leaving
+          active.delete(name)
+          next
+        end
+        raise ValidationError, "Sitemap cycle detected: #{name}" if active.include?(name)
+        next if visited.include?(name)
+
+        active.add(name)
+        visited.add(name)
+        pending << [name, true]
+        children = read_sitemap(name) { |entry| yield entry }
+        children.reverse_each { |child| pending << [child, false] }
       end
     end
 
-    def read_xml_document(filename)
-      document = Nokogiri::XML(storage.read(filename)) { |config| config.strict }
-      document.remove_namespaces!
-      document
+    def read_sitemap(filename)
+      children = []
+      root = nil
+      count = 0
+      open_sitemap(filename) do |io|
+        reader = Nokogiri::XML::Reader.from_io(io, nil, nil, Nokogiri::XML::ParseOptions::NONET)
+        reader.each do |node|
+          if node.node_type == Nokogiri::XML::Reader::TYPE_DOCUMENT_TYPE
+            raise ValidationError, "Sitemap DTDs are not allowed: #{filename}"
+          end
+          next unless node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT
+
+          if node.depth == 0
+            root = node.local_name
+            unless %w[urlset sitemapindex].include?(root)
+              raise ValidationError, "Invalid sitemap root element in #{filename}: #{root}"
+            end
+            unless node.namespace_uri == "http://www.sitemaps.org/schemas/sitemap/0.9"
+              raise ValidationError, "Invalid sitemap namespace: #{filename}"
+            end
+          elsif node.depth == 1 && node.local_name == ((root == "urlset") ? "url" : "sitemap")
+            count += 1
+            raise ValidationError, "Sitemap exceeds 50,000 entries: #{filename}" if count > MAX_ENTRIES
+
+            fields = read_fields(reader)
+            location = location_for(fields, filename: filename)
+            if root == "urlset"
+              lastmod = fields.fetch("lastmod").to_s.strip
+              yield Parser::Entry.new(loc: location, lastmod: lastmod.empty? ? nil : lastmod, source_sitemap: filename)
+            else
+              child = local_child_filename(location, parent_filename: filename)
+              children << child
+            end
+          end
+        end
+      end
+      if count == 0
+        message = (root == "sitemapindex") ? "Sitemap index has no child sitemap URLs" : "Sitemap has no URLs"
+        raise ValidationError, "#{message}: #{filename}"
+      end
+      children.each { |child| raise ValidationError, "Missing child sitemap file: #{child}" unless storage.exist?(child) }
+      if children.uniq.size != children.size
+        raise ValidationError, "Duplicate child sitemap URLs detected: #{filename}"
+      end
+      children
     rescue Nokogiri::XML::SyntaxError => error
       raise ValidationError, "Invalid sitemap XML in #{filename}: #{error.message.lines.first.strip}"
     end
 
-    def validate_urlset_document!(document, sitemap_filename)
-      return if document.xpath("/urlset/url/loc").any?
+    def open_sitemap(filename, &block)
+      if storage.respond_to?(:open) && storage.respond_to?(:metadata)
+        if storage.metadata(filename).fetch(:bytes) > MAX_BYTES
+          raise ValidationError, "Sitemap exceeds 52,428,800 bytes: #{filename}"
+        end
+        storage.open(filename, &block)
+      else
+        body = storage.read(filename)
+        raise ValidationError, "Sitemap exceeds 52,428,800 bytes: #{filename}" if body.bytesize > MAX_BYTES
 
-      raise ValidationError, "Sitemap has no URLs: #{sitemap_filename}"
+        yield StringIO.new(body)
+      end
     end
 
-    def validate_sitemap_index_document!(document, sitemap_filename)
-      child_locations = document.xpath("/sitemapindex/sitemap/loc").map { |node| node.text.to_s.strip }.reject(&:empty?)
-      raise ValidationError, "Sitemap index has no child sitemap URLs: #{sitemap_filename}" if child_locations.empty?
-
-      duplicate_children = child_locations.group_by(&:itself).select { |_loc, values| values.size > 1 }.keys
-      unless duplicate_children.empty?
-        raise ValidationError, "Duplicate child sitemap URLs detected: #{duplicate_children.first(5).join(", ")}"
+    def read_fields(reader)
+      fields = {"loc" => nil, "lastmod" => nil}
+      return fields if reader.self_closing?
+
+      text = nil
+      while reader.read
+        type = reader.node_type
+        if type == Nokogiri::XML::Reader::TYPE_END_ELEMENT
+          break if reader.depth == 1
+
+          text = nil if reader.depth == 2
+        elsif type == Nokogiri::XML::Reader::TYPE_ELEMENT && reader.depth == 2
+          name = reader.local_name
+          if fields.key?(name)
+            raise ValidationError, "Repeated sitemap field: #{name}" if fields[name]
+
+            fields[name] = +""
+            text = reader.self_closing? ? nil : fields[name]
+          else
+            text = nil
+          end
+        elsif text && [Nokogiri::XML::Reader::TYPE_TEXT, Nokogiri::XML::Reader::TYPE_CDATA,
+          Nokogiri::XML::Reader::TYPE_WHITESPACE, Nokogiri::XML::Reader::TYPE_SIGNIFICANT_WHITESPACE].include?(type)
+          text << reader.value
+        end
       end
+      fields
+    end
 
-      child_locations.each do |location|
-        child_filename = local_child_filename(location, parent_filename: sitemap_filename)
-        raise ValidationError, "Missing child sitemap file: #{child_filename}" unless storage.exist?(child_filename)
-
-        validate_sitemap_file!(child_filename)
+    def location_for(fields, filename:)
+      location = fields.fetch("loc").to_s.strip
+      if location.empty?
+        raise ValidationError, "Sitemap entry must have one nonempty loc: #{filename}"
       end
+      raise ValidationError, "Sitemap loc must be shorter than 2,048 characters: #{filename}" if location.length >= 2048
+
+      location
     end
 
     def local_child_filename(location, parent_filename:)
       uri = URI.parse(location)
+      base_uri = URI.parse(configuration.base_url)
+      if uri.query || uri.fragment || uri.userinfo || (uri.host &&
+          [uri.scheme, uri.host, uri.port] != [base_uri.scheme, base_uri.host, base_uri.port]) ||
+          (uri.scheme && !%w[http https].include?(uri.scheme))
+        raise ValidationError, "Invalid child sitemap URL: #{location}"
+      end
       if uri.absolute? || location.start_with?("/")
         normalize_local_filename(uri.path)
       else
@@ -90,29 +207,18 @@ module Indexmap
         normalize_local_filename((parent_directory == ".") ? location : File.join(parent_directory, location))
       end
     rescue URI::InvalidURIError
-      normalize_local_filename(location)
+      raise ValidationError, "Invalid child sitemap URL: #{location}"
     end
 
     def normalize_local_filename(filename)
       normalized = Pathname(filename.to_s).cleanpath.to_s.sub(%r{\A/+}, "")
-      return if normalized.empty? || normalized == ".." || normalized.start_with?("../")
+      if normalized.empty? || normalized == ".." || normalized.start_with?("../")
+        raise ValidationError, "Invalid child sitemap filename: #{filename}"
+      end
 
       normalized
     end
 
-    def validate_presence!(entries)
-      return unless entries.empty?
-
-      raise ValidationError, "Sitemap has no URLs"
-    end
-
-    def validate_duplicates!(entries)
-      duplicates = entries.map(&:loc).group_by(&:itself).select { |_url, values| values.size > 1 }.keys
-      return if duplicates.empty?
-
-      raise ValidationError, "Duplicate sitemap URLs detected: #{duplicates.first(5).join(", ")}"
-    end
-
     def validate_parameterized_urls!(entries)
       param_urls = entries.map(&:loc).select { |url| url&.include?("?") }
       return if param_urls.empty?
@@ -130,7 +236,7 @@ module Indexmap
     def validate_absolute_http_urls!(entries)
       invalid_urls = entries.map(&:loc).reject do |url|
         uri = parse_uri(url)
-        uri&.absolute? && %w[http https].include?(uri.scheme)
+        uri&.absolute? && %w[http https].include?(uri.scheme) && uri.host && !uri.userinfo
       end
       return if invalid_urls.empty?
 
@@ -154,7 +260,11 @@ module Indexmap
       invalid_entries = entries.select do |entry|
         next false if entry.lastmod.nil?
 
-        Date.iso8601(entry.lastmod)
+        if entry.lastmod.match?(/\A\d{4}-\d{2}-\d{2}\z/)
+          Date.iso8601(entry.lastmod)
+        else
+          Time.iso8601(entry.lastmod)
+        end
         false
       rescue ArgumentError
         true
@@ -171,3 +281,5 @@ module Indexmap
     end
   end
 end
+
+require_relative "validator/duplicates"

data/lib/indexmap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Indexmap
-  VERSION = "0.8.2"
+  VERSION = "0.9.0"
 end

data/lib/indexmap/writer.rb

@@ -2,34 +2,68 @@
 
 module Indexmap
   class Writer
+    attr_reader :counts
+
     VALID_FORMATS = %i[index single_file].freeze
 
-    def initialize(base_url:, sections: nil, entries: nil, index_filename: "sitemap.xml", format: :index)
-      @entries = normalize_entries(entries)
+    def initialize(base_url:, sections: nil, entries: nil, index_filename: "sitemap.xml", format: :index, max_urls: 50_000, max_bytes: 52_428_800, generation: nil)
+      @entries = entries || []
       @format = normalize_format(format)
       @sections = normalize_sections(sections)
       @base_url = base_url
       @index_filename = index_filename
+      @max_urls = Integer(max_urls.to_s, 10)
+      @max_bytes = Integer(max_bytes.to_s, 10)
+      @generation = generation
+      @counts = {}
+      unless (1..Validator::MAX_ENTRIES).cover?(@max_urls) && (1..Validator::MAX_BYTES).cover?(@max_bytes)
+        raise ConfigurationError, "Invalid sitemap partition limits"
+      end
     end
 
     def write
-      return [write_file(index_filename, urlset_xml(entries))] if single_file?
+      each_file.to_a
+    end
 
-      paths = sections.map do |section|
-        write_file(section.filename, urlset_xml(section.entries))
-      end
+    def each_file
+      return enum_for(:each_file) unless block_given?
 
-      paths + [write_file(index_filename, index_xml(sections))]
+      if single_file?
+        body, = urlset_xml(entries)
+      else
+        index_entries = []
+        sections.each_with_index do |section, section_index|
+          part = 0
+          chunks(section.entries) do |body, lastmod, count|
+            part += 1
+            filename = if @generation
+              Publication.new(root: index_filename).filename(generation: @generation, section: section_index + 1, part: part)
+            elsif part == 1
+              section.filename
+            else
+              section.filename.sub(/\.xml\z/, "-#{Kernel.format("%03d", part)}.xml")
+            end
+            raise ConfigurationError, "Section filenames must end in .xml" if part > 1 && filename == section.filename
+
+            @counts[filename] = count
+            index_entries << [filename, lastmod]
+            raise ValidationError, "Sitemap exceeds 50,000 entries: #{index_filename}" if index_entries.size > Validator::MAX_ENTRIES
+
+            yield write_file(filename, body)
+          end
+        end
+        body = index_xml(index_entries)
+        raise ValidationError, "Sitemap exceeds 52,428,800 bytes: #{index_filename}" if body.bytesize > @max_bytes
+
+        @counts[index_filename] = index_entries.size
+      end
+      yield write_file(index_filename, body)
     end
 
     private
 
     attr_reader :base_url, :entries, :format, :index_filename, :sections
 
-    def normalize_entries(raw_entries)
-      Array(raw_entries).map { |entry| normalize_entry(entry) }
-    end
-
     def normalize_format(value)
       normalized = value.nil? ? :index : value.to_sym
       return normalized if VALID_FORMATS.include?(normalized)
@@ -53,25 +87,52 @@ module Indexmap
     end
 
     def write_file(filename, body)
-      Storage::File.new(filename: filename, body: body, content_type: "application/xml")
+      Artifact.new(filename: filename, body: body, content_type: "application/xml")
+    end
+
+    HEADER = %(<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n)
+    FOOTER = "</urlset>\n"
+
+    def chunks(source)
+      body = +HEADER
+      latest = nil
+      count = 0
+      source.each do |value|
+        entry = normalize_entry(value)
+        xml = "  <url>\n    <loc>#{escape(entry.loc)}</loc>\n"
+        timestamp = nil
+        if entry.lastmod
+          date = parsed_lastmod(entry.lastmod)
+          xml << "    <lastmod>#{escape(date.iso8601)}</lastmod>\n"
+          timestamp = comparable_lastmod(date)
+        end
+        xml << "  </url>\n"
+        if HEADER.bytesize + xml.bytesize + FOOTER.bytesize > @max_bytes
+          raise ValidationError, "Sitemap entry exceeds byte limit"
+        end
+        if count > 0 && (count == @max_urls || body.bytesize + xml.bytesize + FOOTER.bytesize > @max_bytes)
+          yield body << FOOTER, latest, count
+          body = +HEADER
+          latest = nil
+          count = 0
+        end
+        body << xml
+        latest = timestamp if timestamp && (latest.nil? || timestamp > latest)
+        count += 1
+      end
+      yield body << FOOTER, latest, count if count > 0
     end
 
-    def urlset_xml(entries)
-      lines = [
-        %(<?xml version="1.0" encoding="UTF-8"?>),
-        %(<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">)
-      ]
+    def urlset_xml(source)
+      result = nil
+      chunks(source) do |body, lastmod, count|
+        raise ValidationError, "Single-file sitemap exceeds partition limits; use index format" if result
 
-      entries.each do |entry|
-        sitemap_entry = normalize_entry(entry)
-        lines << "  <url>"
-        lines << "    <loc>#{escape(sitemap_entry.loc)}</loc>"
-        lines << "    <lastmod>#{format_lastmod(sitemap_entry.lastmod)}</lastmod>" if sitemap_entry.lastmod
-        lines << "  </url>"
+        @counts[index_filename] = count
+        result = [body, lastmod]
       end
-
-      lines << "</urlset>"
-      lines.join("\n") + "\n"
+      @counts[index_filename] ||= 0
+      result || [HEADER + FOOTER, nil]
     end
 
     def index_xml(sitemap_sections)
@@ -80,10 +141,9 @@ module Indexmap
         %(<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">)
       ]
 
-      sitemap_sections.each do |section|
+      sitemap_sections.each do |filename, lastmod|
         lines << "  <sitemap>"
-        lines << "    <loc>#{escape(index_loc(section.filename))}</loc>"
-        lastmod = section_lastmod(section)
+        lines << "    <loc>#{escape(index_loc(filename))}</loc>"
         lines << "    <lastmod>#{format_lastmod(lastmod)}</lastmod>" if lastmod
         lines << "  </sitemap>"
       end
@@ -102,11 +162,6 @@ module Indexmap
       File.join(base_url.sub(%r{/\z}, ""), filename)
     end
 
-    def section_lastmod(section)
-      timestamps = Array(section.entries).map { |entry| comparable_lastmod(normalize_entry(entry).lastmod) }.compact
-      timestamps.max
-    end
-
     def format_lastmod(value)
       timestamp = parsed_lastmod(value)
 

data/lib/indexmap.rb

@@ -8,11 +8,13 @@ require "time"
 require_relative "indexmap/version"
 require_relative "indexmap/google_configuration"
 require_relative "indexmap/index_now_configuration"
-require_relative "indexmap/storage/file"
+require_relative "indexmap/robots"
+require_relative "indexmap/artifact"
 require_relative "indexmap/storage/memory"
 require_relative "indexmap/storage/filesystem"
 require_relative "indexmap/storage/active_storage"
 require_relative "indexmap/configuration"
+require_relative "indexmap/publication"
 require_relative "indexmap/creator"
 require_relative "indexmap/entry"
 require_relative "indexmap/output"
@@ -33,6 +35,17 @@ module Indexmap
 
   class ConfigurationError < Error; end
   class ValidationError < Error; end
+  class SupersededPublication < Error; end
+
+  class PublicationCleanupError < Error
+    attr_reader :failures
+
+    def initialize(failures)
+      @failures = failures
+      artifacts = failures.map { |failure| failure.fetch(:artifact) }.join(", ")
+      super("Failed to clean up publication artifacts: #{artifacts}")
+    end
+  end
 
   class << self
     def configuration

data/lib/tasks/indexmap_tasks.rake

@@ -5,7 +5,7 @@ namespace :indexmap do
       runner = Indexmap::TaskRunner.new
       create_result = runner.create
 
-      puts "Created, formatted, and validated #{file_count(create_result[:files])}."
+      puts "Created and validated #{file_count(create_result[:files])}."
       puts "Files created:"
       create_result[:sitemaps].each do |sitemap|
         puts "  - #{sitemap[:location]} (#{link_count(sitemap[:link_count])})"
@@ -13,6 +13,13 @@ namespace :indexmap do
       puts "IndexNow key file: #{create_result[:index_now_key_filename]}" if create_result[:index_now_key_filename]
     end
 
+    desc "Prune expired immutable generations (RETENTION_HOURS defaults to 48)"
+    task prune: :environment do
+      hours = Integer(ENV.fetch("RETENTION_HOURS", "48"), 10)
+      deleted = Indexmap::TaskRunner.new.prune(retention: hours * 3600)
+      puts "Pruned #{deleted.size} sitemap artifacts."
+    end
+
     desc "Format sitemap files for better readability"
     task format: :environment do
       runner = Indexmap::TaskRunner.new
@@ -56,9 +63,21 @@ namespace :indexmap do
   end
 
   namespace :index_now do
+    desc "Preview exact IndexNow selection as JSON Lines without submitting"
+    task plan: :environment do
+      Indexmap::Pinger::IndexNow.new.plan.each { |row| puts JSON.generate(row) }
+    end
+
     desc "Ping IndexNow. ENV: SINCE=2026-04-18T10:30:00Z or INDEXNOW_RECENT_HOURS=24"
     task ping: :environment do
-      result = Indexmap::Pinger::IndexNow.new.ping
+      report = ENV["INDEXNOW_REPORT"] == "1"
+      result = Indexmap::Pinger::IndexNow.new.ping do |event|
+        puts JSON.generate(event) if report
+      end
+      if report
+        puts JSON.generate(result.merge(event: :summary))
+        next
+      end
 
       case result[:status]
       when :submitted