site_maps 0.0.1.beta1

data/lib/site_maps/adapters/aws_sdk/storage.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+class SiteMaps::Adapters::AwsSdk::Storage
+  attr_reader :config
+
+  def initialize(config)
+    @config = config
+  end
+
+  def upload(location, **options)
+    options[:acl] ||= config.acl if config.acl
+    options[:cache_control] ||= config.cache_control if config.cache_control
+    options[:content_type] ||= location.gzip? ? "application/gzip" : "application/xml"
+    lastmod = options.delete(:last_modified) || Time.now
+    options[:metadata] ||= {}
+    options[:metadata]["given-last-modified"] = lastmod.utc.strftime("%Y-%m-%dT%H:%M:%S%:z")
+    obj = object(location.remote_path)
+    obj.upload_file(location.path, **options)
+  end
+
+  def read(location)
+    obj = object(location.remote_path).get
+    metadata = {
+      content_type: obj.content_type
+    }
+    if (raw = obj.metadata["given-last-modified"]) &&
+        (time = Time.parse(raw))
+      metadata[:last_modified] = time
+    end
+    [obj.body.read, metadata]
+  rescue Aws::S3::Errors::NoSuchKey
+    raise SiteMaps::FileNotFoundError, "File not found: #{location.remote_path}"
+  end
+
+  def delete(location)
+    object(location.remote_path).delete
+  rescue Aws::S3::Errors::NoSuchKey
+    raise SiteMaps::FileNotFoundError, "File not found: #{location.remote_path}"
+  end
+
+  private
+
+  def list_objects(prefix:)
+    config.s3_bucket.objects(
+      prefix: prefix
+    )
+  end
+
+  def object(remote_path)
+    config.s3_bucket.object(remote_path)
+  end
+end

data/lib/site_maps/adapters/aws_sdk.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SiteMaps::Adapters
+  class AwsSdk < Adapter
+    def write(url, raw_data, **options)
+      location = Location.new(config.directory, url)
+      local_storage.write(location, raw_data)
+      s3_storage.upload(location, **options)
+    end
+
+    def read(url)
+      location = Location.new(config.directory, url)
+      s3_storage.read(location)
+    end
+
+    def delete(url)
+      location = Location.new(config.directory, url)
+      s3_storage.delete(location)
+    end
+
+    private
+
+    def local_storage
+      @local_storage ||= SiteMaps::Adapters::FileSystem::Storage.new
+    end
+
+    def s3_storage
+      @s3_storage ||= SiteMaps::Adapters::AwsSdk::Storage.new(config)
+    end
+  end
+end

data/lib/site_maps/adapters/file_system/config.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class SiteMaps::Adapters::FileSystem::Config < SiteMaps::Configuration
+  attribute :directory, default: "public/sitemaps"
+end

data/lib/site_maps/adapters/file_system/location.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class SiteMaps::Adapters::FileSystem::Location < Struct.new(:root, :url)
+  ROOT_RE = %r{^/}
+  GZIP_RE = %r{\.gz$}
+
+  def path
+    File.join(
+      root,
+      make_relative(uri.path)
+    )
+  end
+
+  def directory
+    Pathname.new(root).join(remote_relative_dir).to_s
+  end
+
+  def gzip?
+    GZIP_RE.match?(uri.path)
+  end
+
+  private
+
+  def uri
+    @uri ||= URI.parse(url)
+  end
+
+  def remote_relative_dir
+    make_relative(File.dirname(uri.path))
+  end
+
+  def make_relative(path)
+    path.sub(ROOT_RE, "")
+  end
+end

data/lib/site_maps/adapters/file_system/storage.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+class SiteMaps::Adapters::FileSystem::Storage
+  # @param [SiteMaps::Adapters::FileSystem::Location] location
+  # @param [String] raw_data
+  # @return [void]
+  # @raise [SiteMaps::Error] if the path is not a directory
+  def write(location, raw_data, **)
+    dir = location.directory
+
+    if !File.exist?(dir)
+      FileUtils.mkdir_p(dir)
+    elsif !File.directory?(dir)
+      raise SiteMaps::Error.new("The path #{dir} is not a directory")
+    end
+
+    stream = File.open(location.path, "wb")
+    if location.gzip?
+      gzip(stream, raw_data)
+    else
+      plain(stream, raw_data)
+    end
+  end
+
+  # @param [SiteMaps::Adapters::FileSystem::Location] location
+  # @return [Array<String, Hash>] The raw data and metadata
+  # @raise [SiteMaps::FileNotFoundError] if the file does not exist
+  def read(location)
+    if location.gzip?
+      [Zlib::GzipReader.open(location.path).read, {content_type: "application/gzip"}]
+    else
+      [File.read(location.path), {content_type: "application/xml"}]
+    end
+  rescue Zlib::GzipFile::Error
+    raise SiteMaps::FileNotFoundError.new("File not found: #{location.path}")
+  rescue Errno::ENOENT
+    raise SiteMaps::FileNotFoundError.new("File not found: #{location.path}")
+  end
+
+  # @param [SiteMaps::Adapters::FileSystem::Location] location
+  # @return [void]
+  # @raise [SiteMaps::FileNotFoundError] if the file does not exist
+  def delete(location)
+    File.delete(location.path)
+  rescue Errno::ENOENT
+    raise SiteMaps::FileNotFoundError.new("File not found: #{location.path}")
+  end
+
+  protected
+
+  def gzip(stream, data)
+    gz = Zlib::GzipWriter.new(stream)
+    gz.write data
+    gz.close
+  end
+
+  def plain(stream, data)
+    stream.write data
+    stream.close
+  end
+end

data/lib/site_maps/adapters/file_system.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module SiteMaps::Adapters
+  class FileSystem < Adapter
+    def write(url, raw_data, **)
+      location = Location.new(config.directory, url)
+      storage.write(location, raw_data)
+    end
+
+    def read(url)
+      location = Location.new(config.directory, url)
+      storage.read(location)
+    end
+
+    def delete(url)
+      location = Location.new(config.directory, url)
+      storage.delete(location)
+    end
+
+    private
+
+    def storage
+      @storage ||= self.class::Storage.new
+    end
+  end
+end

data/lib/site_maps/adapters/noop.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SiteMaps::Adapters
+  class Noop < Adapter
+    def write(_url, _raw_data, **_kwargs)
+    end
+
+    def read(_url)
+    end
+
+    def delete(_url)
+    end
+
+    def fetch_sitemap_index_links
+      []
+    end
+  end
+end

data/lib/site_maps/atomic_repository.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  class AtomicRepository
+    attr_reader :main_url, :preloaded_index_links
+
+    def initialize(main_url)
+      @main_url = main_url
+      @preloaded_index_links = Concurrent::Array.new
+      @generated_urls = Concurrent::Hash.new
+    end
+
+    def generate_url(raw_location)
+      location = IncrementalLocation.new(main_url, raw_location)
+      (@generated_urls[location.relative_directory] ||= location).next.url
+    end
+
+    def remaining_index_links
+      preloaded_index_links.reject do |link|
+        @generated_urls.key?(link.relative_directory)
+      end
+    end
+  end
+end

data/lib/site_maps/builder/link.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SiteMaps::Builder
+  class Link
+    attr_reader :uri
+
+    def initialize(base_url, path, params = {})
+      @uri = base_url.is_a?(::URI) ? base_url.dup : ::URI.parse(base_url)
+      @uri.user, @uri.query = nil
+      @uri.path = path
+      @uri.query = Rack::Utils.unescape(Rack::Utils.build_nested_query(params)) if params.is_a?(Hash) && params.any?
+    end
+
+    def to_s
+      @uri.to_s
+    end
+
+    def eql?(other)
+      to_s == other.to_s
+    end
+    alias_method :==, :eql?
+
+    def hash
+      to_s.hash
+    end
+  end
+end

data/lib/site_maps/builder/normalizer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SiteMaps::Builder
+  module Normalizer
+    extend self
+
+    def format_float(value)
+      value.is_a?(String) ? value : ("%0.1f" % value)
+    end
+
+    def yes_or_no(value)
+      if value.is_a?(String) && value.match?(/\A(yes|no)\z/i)
+        value.downcase
+      else
+        value ? "yes" : "no"
+      end
+    end
+
+    def yes_or_no_with_default(value, default)
+      value.nil? ? yes_or_no(default) : yes_or_no(value)
+    end
+
+    def w3c_date(date)
+      if date.is_a?(String)
+        date
+      elsif date.respond_to?(:iso8601)
+        date.iso8601.sub(/Z$/i, "+00:00")
+      elsif date.is_a?(Date) && defined?(DateTime) && !date.is_a?(DateTime)
+        date.strftime("%Y-%m-%d")
+      else
+        zulutime = if defined?(DateTime) && date.is_a?(DateTime)
+          date.new_offset(0)
+        elsif date.respond_to?(:utc)
+          date.utc
+        elsif date.is_a?(Integer)
+          Time.at(date).utc
+        end
+
+        if zulutime
+          zulutime.strftime("%Y-%m-%dT%H:%M:%S+00:00")
+        else
+          zone = date.strftime("%z").insert(-3, ":")
+          date.strftime("%Y-%m-%dT%H:%M:%S") + zone
+        end
+      end
+    end
+  end
+end

data/lib/site_maps/builder/sitemap_index/item.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class SiteMaps::Builder::SitemapIndex::Item < Struct.new(:loc, :lastmod)
+  extend Forwardable
+
+  def to_xml
+    builder = ::Builder::XmlMarkup.new
+    builder.sitemap do
+      builder.loc(loc)
+      builder.lastmod w3c_date(lastmod) if lastmod
+    end
+    builder << "\n"
+  end
+
+  def eql?(other)
+    loc == other.loc
+  end
+  alias_method :==, :eql?
+
+  def hash
+    loc.hash
+  end
+
+  def relative_directory
+    return unless loc =~ %r{^https?://[^/]+(/.*)$}
+
+    val = File.dirname(Regexp.last_match(1))
+    val = val[1..-1] if val.start_with?("/")
+    val
+  end
+
+  protected
+
+  def_delegator SiteMaps::Builder::Normalizer, :w3c_date
+end

data/lib/site_maps/builder/sitemap_index.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module SiteMaps::Builder
+  class SitemapIndex
+    HEADER = <<~HEADER
+      <?xml version="1.0" encoding="UTF-8"?>
+      <sitemapindex
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/siteindex.xsd"
+        xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+      >
+    HEADER
+    FOOTER = "</sitemapindex>"
+
+    attr_reader :sitemaps
+
+    def initialize
+      @sitemaps = Concurrent::Set.new
+    end
+
+    def add(loc, lastmod: nil)
+      sitemap = loc.is_a?(Item) ? loc : Item.new(loc, lastmod)
+      @sitemaps.add(sitemap)
+    end
+
+    def to_xml
+      io = StringIO.new
+      io.puts(HEADER)
+      @sitemaps.each do |sitemap|
+        io.puts(sitemap.to_xml)
+      end
+      io.puts(FOOTER)
+      io.string
+    end
+
+    def empty?
+      @sitemaps.empty?
+    end
+  end
+end

data/lib/site_maps/builder/url.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module SiteMaps::Builder
+  class URL
+    extend Forwardable
+
+    DEFAULTS = {
+      changefreq: "weekly",
+      priority: 0.5
+    }.freeze
+
+    attr_reader :attributes
+
+    def initialize(link, **attributes)
+      @attributes = DEFAULTS.merge(attributes)
+      @attributes[:loc] = link
+      @attributes[:alternates] = SiteMaps::Primitives::Array.wrap(@attributes[:alternates])
+      @attributes[:videos] = SiteMaps::Primitives::Array.wrap(@attributes[:videos])
+      @attributes[:images] = SiteMaps::Primitives::Array.wrap(@attributes[:images])
+      if (video = @attributes.delete(:video))
+        @attributes[:videos].concat(SiteMaps::Primitives::Array.wrap(video))
+      end
+      if (alternate = @attributes.delete(:alternate))
+        @attributes[:alternates].concat(SiteMaps::Primitives::Array.wrap(alternate))
+      end
+      if (image = @attributes.delete(:image))
+        @attributes[:images].concat(SiteMaps::Primitives::Array.wrap(image))
+      end
+      @attributes[:images] = @attributes[:images][0...SiteMaps::MAX_LENGTH[:images]]
+    end
+
+    def [](key)
+      attributes[key]
+    end
+
+    def last_modified
+      return unless self[:lastmod].respond_to?(:strftime)
+
+      self[:lastmod]
+    end
+
+    def to_xml
+      return @to_xml if defined?(@to_xml)
+
+      builder = ::Builder::XmlMarkup.new
+      builder.url do
+        builder.loc self[:loc]
+        builder.lastmod w3c_date(self[:lastmod]) if self[:lastmod]
+        builder.expires w3c_date(self[:expires]) if self[:expires]
+        builder.changefreq self[:changefreq].to_s if self[:changefreq]
+        builder.priority format_float(self[:priority]) if self[:priority]
+
+        if news?
+          news_data = self[:news]
+          builder.news :news do
+            builder.news :publication do
+              builder.news :name, news_data[:publication_name].to_s if news_data[:publication_name]
+              builder.news :language, news_data[:publication_language].to_s if news_data[:publication_language]
+            end
+
+            builder.news :access, news_data[:access].to_s if news_data[:access]
+            builder.news :genres, news_data[:genres].to_s if news_data[:genres]
+            builder.news :publication_date, w3c_date(news_data[:publication_date]) if news_data[:publication_date]
+            builder.news :title, news_data[:title].to_s if news_data[:title]
+            builder.news :keywords, news_data[:keywords].to_s if news_data[:keywords]
+            builder.news :stock_tickers, news_data[:stock_tickers].to_s if news_data[:stock_tickers]
+          end
+        end
+
+        self[:images].each do |image|
+          builder.image :image do
+            builder.image :loc, image[:loc]
+            builder.image :caption, image[:caption].to_s if image[:caption]
+            builder.image :geo_location, image[:geo_location].to_s if image[:geo_location]
+            builder.image :title, image[:title].to_s if image[:title]
+            builder.image :license, image[:license].to_s if image[:license]
+          end
+        end
+
+        self[:videos].each do |video|
+          builder.video :video do
+            builder.video :thumbnail_loc, video[:thumbnail_loc].to_s
+            builder.video :title, video[:title].to_s
+            builder.video :description, video[:description].to_s
+            builder.video :content_loc, video[:content_loc].to_s if video[:content_loc]
+            if video[:player_loc]
+              loc_attributes = {allow_embed: yes_or_no_with_default(video[:allow_embed], true)}
+              loc_attributes[:autoplay] = video[:autoplay].to_s if video[:autoplay]
+              builder.video :player_loc, video[:player_loc].to_s, loc_attributes
+            end
+            builder.video :duration, video[:duration].to_s if video[:duration]
+            builder.video :expiration_date, w3c_date(video[:expiration_date]) if video[:expiration_date]
+            builder.video :rating, format_float(video[:rating]) if video[:rating]
+            builder.video :view_count, video[:view_count].to_s if video[:view_count]
+            builder.video :publication_date, w3c_date(video[:publication_date]) if video[:publication_date]
+            video[:tags]&.each { |tag| builder.video :tag, tag.to_s }
+            builder.video :tag, video[:tag].to_s if video[:tag]
+            builder.video :category, video[:category].to_s if video[:category]
+            builder.video :family_friendly, yes_or_no_with_default(video[:family_friendly], true) if video.has_key?(:family_friendly)
+            builder.video :gallery_loc, video[:gallery_loc].to_s, title: video[:gallery_title].to_s if video[:gallery_loc]
+            builder.video :price, video[:price].to_s, prepare_video_price_attribs(video) if video[:price]
+            if video[:uploader]
+              builder.video :uploader, video[:uploader].to_s, video[:uploader_info] ? {info: video[:uploader_info].to_s} : {}
+            end
+            builder.video :live, yes_or_no_with_default(video[:live], true) if video.has_key?(:live)
+            builder.video :requires_subscription, yes_or_no_with_default(video[:requires_subscription], true) if video.has_key?(:requires_subscription)
+          end
+        end
+
+        self[:alternates].each do |alternate|
+          rel = alternate[:nofollow] ? "alternate nofollow" : "alternate"
+          attributes = {rel: rel, href: alternate[:href].to_s}
+          attributes[:hreflang] = alternate[:lang].to_s if alternate[:lang]
+          attributes[:media] = alternate[:media].to_s if alternate[:media]
+          builder.xhtml :link, attributes
+        end
+
+        unless self[:mobile].nil?
+          builder.mobile :mobile
+        end
+
+        if self[:pagemap].is_a?(Hash) && (pagemap = self[:pagemap]).any?
+          builder.pagemap :PageMap do
+            SiteMaps::Primitives::Array.wrap(pagemap[:dataobjects]).each do |dataobject|
+              builder.pagemap :DataObject, type: dataobject[:type].to_s, id: dataobject[:id].to_s do
+                SiteMaps::Primitives::Array.wrap(dataobject[:attributes]).each do |attribute|
+                  builder.pagemap :Attribute, attribute[:value].to_s, name: attribute[:name].to_s
+                end
+              end
+            end
+          end
+        end
+      end
+      @to_xml = builder << "\n"
+    end
+
+    def news?
+      self[:news].is_a?(Hash) && self[:news].any?
+    end
+
+    def bytesize
+      to_xml.bytesize
+    end
+
+    private
+
+    def_delegator SiteMaps::Builder::Normalizer, :format_float
+    def_delegator SiteMaps::Builder::Normalizer, :yes_or_no
+    def_delegator SiteMaps::Builder::Normalizer, :yes_or_no_with_default
+    def_delegator SiteMaps::Builder::Normalizer, :w3c_date
+  end
+end

data/lib/site_maps/builder/url_set.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module SiteMaps::Builder
+  class URLSet
+    SCHEMAS = {
+      "image" => "http://www.google.com/schemas/sitemap-image/1.1",
+      "mobile" => "http://www.google.com/schemas/sitemap-mobile/1.0",
+      "news" => "http://www.google.com/schemas/sitemap-news/0.9",
+      "pagemap" => "http://www.google.com/schemas/sitemap-pagemap/1.0",
+      "video" => "http://www.google.com/schemas/sitemap-video/1.1"
+    }.freeze
+
+    HEADER = <<~HEADER
+      <?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml"
+        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"
+      #{SCHEMAS.map { |name, uri| "  xmlns:#{name}=\"#{uri}\"" }.join("\n")}
+      >
+    HEADER
+    FOOTER = "</urlset>"
+    FOOTER_BYTESIZE = FOOTER.bytesize
+
+    attr_reader :content, :links_count, :news_count
+
+    def initialize
+      @content = StringIO.new
+      @content.puts(HEADER)
+      @links_count = 0
+      @news_count = 0
+      @last_modified = nil
+    end
+
+    def add(link, **options)
+      raise SiteMaps::FullSitemapError if finalized?
+
+      url = SiteMaps::Builder::URL.new(link, **options)
+      raise SiteMaps::FullSitemapError unless fit?(url)
+
+      content.puts(url.to_xml)
+      @links_count += 1
+      @news_count += 1 if url.news?
+      if (lastmod = url.last_modified)
+        @last_modified ||= lastmod
+        @last_modified = lastmod if lastmod > @last_modified
+      end
+      url
+    end
+
+    def finalize!
+      return if finalized?
+
+      content.puts(FOOTER)
+      @to_xml = content.string.freeze
+      content.close
+      @to_xml
+    end
+
+    def to_xml
+      return content.string + FOOTER unless finalized?
+
+      @to_xml
+    end
+
+    def finalized?
+      defined?(@to_xml)
+    end
+
+    def empty?
+      links_count.zero?
+    end
+
+    def last_modified
+      @last_modified || Time.now
+    end
+
+    private
+
+    def bytesize
+      content.string.bytesize
+    end
+
+    # @param url [Builder::URL]
+    def fit?(url)
+      return false if links_count >= SiteMaps::MAX_LENGTH[:links]
+      return false if url.news? && news_count >= SiteMaps::MAX_LENGTH[:news]
+
+      (bytesize + url.bytesize + FOOTER_BYTESIZE) <= SiteMaps::MAX_FILESIZE
+    end
+  end
+end