site_maps 0.0.1.beta1

data/lib/site_maps/cli.rb

@@ -0,0 +1,68 @@
+require "thor"
+
+module SiteMaps
+  class CLI < Thor
+    method_option :debug, type: :boolean, default: false
+    method_option :logfile, type: :string, default: nil
+    method_option :pidfile, type: :string, default: nil
+    method_option :config_file, type: :string, aliases: "-r", default: nil
+    method_option :max_threads, type: :numeric, aliases: "-c", default: 4
+    method_option :context, type: :hash, default: {}
+    method_option :enqueue_remaining, type: :boolean, default: false
+
+    desc "generate 1st_process,2nd_process ... ,Nth_process", "Generate sitemap.xml files for the given processes"
+    default_command :start
+
+    def generate(processes = "")
+      load_rails if rails_app?
+
+      opts = (@options || {}).transform_keys(&:to_sym)
+      if (logfile = opts[:logfile])
+        SiteMaps.logger = Logger.new(logfile)
+      end
+      if opts[:debug]
+        SiteMaps.logger.level = Logger::DEBUG
+      end
+
+      SiteMaps::Notification.subscribe(SiteMaps::Runner::EventListener)
+
+      runner = SiteMaps.generate(
+        config_file: opts[:config_file],
+        max_threads: opts[:max_threads]
+      )
+      if processes.empty?
+        runner.enqueue_all
+      else
+        kwargs = (opts[:context] || {}).transform_keys(&:to_sym)
+        processes.split(",").each do |process|
+          runner.enqueue(process.strip.to_sym, **kwargs)
+        end
+      end
+      if opts[:enqueue_remaining]
+        runner.enqueue_remaining
+      end
+
+      runner.run
+    end
+
+    desc "version", "Print the version"
+    def version
+      puts "SiteMaps version: #{SiteMaps::VERSION}"
+    end
+
+    default_task :help
+
+    private
+
+    def rails_app?
+      File.exist?(File.join(Dir.pwd, "config", "application.rb"))
+    end
+
+    def load_rails
+      require File.expand_path(File.join(Dir.pwd, "config", "application.rb"))
+      require_relative "railtie"
+
+      ::Rails.application.require_environment!
+    end
+  end
+end

data/lib/site_maps/configuration.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  class Configuration
+    class << self
+      def attributes
+        @attributes || {}
+      end
+
+      def attribute(name, default: nil)
+        @attributes ||= {}
+        @attributes[name] = default
+
+        unless method_defined?(name)
+          define_method(name) do
+            instance_variable_get(:"@#{name}")
+          end
+        end
+
+        unless method_defined?(:"#{name}=")
+          define_method(:"#{name}=") do |value|
+            instance_variable_set(:"@#{name}", value)
+          end
+        end
+
+        unless method_defined?(:"#{name}?")
+          define_method(:"#{name}?") do
+            !!send(name)
+          end
+        end
+      end
+
+      def inherited(subclass)
+        subclass.instance_variable_set(:@attributes, attributes.dup)
+      end
+    end
+
+    attribute :url
+    attribute :directory, default: "/tmp/sitemaps"
+
+    def initialize(**options)
+      default_attributes.merge(options).each do |key, value|
+        send(:"#{key}=", value)
+      rescue NoMethodError
+        raise ConfigurationError, <<~ERROR
+          Unknown configuration option: #{key}
+        ERROR
+      end
+    end
+
+    def becomes(klass, **options)
+      klass.new(**to_h, **options)
+    end
+
+    def to_h
+      instance_variables.each_with_object({}) do |var, hash|
+        hash[var.to_s.delete("@").to_sym] = instance_variable_get(var)
+      end
+    end
+
+    def url
+      @url || validate_url!
+    end
+
+    def base_uri
+      ::URI.parse(url).tap do |uri|
+        uri.path = ""
+        uri.query = nil
+        uri.fragment = nil
+      end
+    end
+
+    def local_sitemap_path
+      filename = ::File.basename(url)
+      Pathname.new(directory).join(filename)
+    end
+
+    def fetch_sitemap_index_links
+      doc = SiteMaps::SitemapReader.new(local_sitemap_path.exist? ? local_sitemap_path : url).to_doc
+
+      doc.css("sitemapindex sitemap").map do |url|
+        SiteMaps::Builder::SitemapIndex::Item.new(
+          url.at_css("loc").text,
+          url.at_css("lastmod")&.text
+        )
+      end
+    rescue SiteMaps::SitemapReader::Error
+      []
+    end
+
+    def remote_sitemap_directory
+      path = ::URI.parse(url).path
+      path = path[1..-1] if path.start_with?("/")
+      path.split("/")[0..-2].join("/")
+    end
+
+    private
+
+    def validate_url!
+      return if @url
+
+      raise ConfigurationError, <<~ERROR
+        You must set a sitemap URL in your configuration to use the add method.
+
+        Example:
+          SiteMaps.configure do |config|
+            config.url = "https://example.com/sitemap.xml"
+          end
+      ERROR
+    end
+
+    def default_attributes
+      self.class.attributes.each_with_object({}) do |(key, default), hash|
+        value = default.respond_to?(:call) ? default.call : default
+        hash[key] = value unless value.nil?
+      end
+    end
+  end
+end

data/lib/site_maps/incremental_location.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  class IncrementalLocation
+    FILENAME = "sitemap.xml"
+    PLACEHOLDER = "%{index}"
+
+    def initialize(main_url, process_location)
+      @main_uri = URI(main_url)
+      @index = Concurrent::AtomicFixnum.new(0)
+      normalize(process_location || @main_uri.to_s)
+    end
+
+    def url
+      placeholder_url % {index: @index.value}
+    end
+
+    def next
+      @index.increment
+      self
+    end
+
+    def main_url
+      main_uri.to_s
+    end
+
+    def relative_directory
+      File.dirname(@uri.path).sub(%r{^/}, "")
+    end
+
+    private
+
+    attr_reader :main_uri, :placeholder_url
+
+    def base_url
+      main_uri.dup.tap { |uri| uri.path = "" }
+    end
+
+    def base_dir
+      File.dirname(main_uri.path)
+    end
+
+    def normalize(loc)
+      uri = if %r{^https?://}.match?(loc)
+        URI(loc)
+      elsif loc.start_with?("/")
+        main_uri.dup.tap { |uri| uri.path = loc }
+      else
+        main_uri.dup.tap { |uri| uri.path = File.join(base_dir, loc) }
+      end
+      unless %w[.xml .xml.gz].include?(File.extname(uri.path))
+        uri.path = File.join(uri.path, FILENAME)
+      end
+      base = uri.dup.tap { |v| v.path = "" }.to_s
+      basename = File.basename(uri.path)
+      index_basename = basename.sub(/[\.](xml|xml\.gz)$/, "#{PLACEHOLDER}.\\1")
+
+      @placeholder_url = File.join(base, File.join(File.dirname(uri.path), index_basename))
+      @uri = URI(File.join(base, File.join(File.dirname(uri.path), basename)))
+    end
+  end
+end

data/lib/site_maps/notification/bus.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  module Notification
+    class Bus
+      attr_reader :listeners, :events
+
+      def initialize
+        @listeners = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
+        @events = Concurrent::Hash.new
+      end
+
+      def publish(event_id, payload)
+        raise UnregisteredEventError, event_id unless can_handle?(event_id)
+
+        process(event_id, payload) do |event, listener|
+          # Concurrent::Future.execute { listener.call(event) }
+          listener.call(event)
+        end
+        self
+      end
+
+      def attach(listener)
+        events.each do |id, event|
+          method_name = event.listener_method
+          next unless listener.respond_to?(method_name)
+
+          listeners[id] << listener.method(method_name)
+        end
+        self
+      end
+
+      def unsubscribe(listener)
+        listeners.each do |id, arr|
+          arr.each do |func|
+            listeners[id].delete(func) if func.receiver == listener
+          end
+        end
+        self
+      end
+      alias_method :detach, :unsubscribe
+
+      def subscribe(object_or_event_id, &block)
+        raise(InvalidSubscriberError, object_or_event_id) unless can_handle?(object_or_event_id)
+
+        if block
+          listeners[object_or_event_id] << block
+        else
+          attach(object_or_event_id)
+        end
+
+        self
+      end
+
+      # rubocop:disable Performance/RedundantEqualityComparisonBlock
+      def subscribed?(listener)
+        listeners.values.any? { |value| value.any? { |func| func == listener } } ||
+          (
+            methods = events.values.map(&:listener_method)
+              .select { |method_name| listener.respond_to?(method_name) }
+              .map { |method_name| listener.method(method_name) }
+            methods && listeners.values.any? { |value| (methods & value).size > 0 }
+          )
+      end
+      # rubocop:enable Performance/RedundantEqualityComparisonBlock
+
+      def can_handle?(object_or_event_id)
+        case object_or_event_id
+        when String, Symbol
+          events.key?(object_or_event_id)
+        else
+          events
+            .values
+            .map(&:listener_method)
+            .any? { |method_name| object_or_event_id.respond_to?(method_name) }
+        end
+      end
+
+      protected
+
+      def process(event_id, payload)
+        listeners[event_id].each do |listener|
+          event = events[event_id].payload(payload)
+
+          yield(event, listener)
+        end
+      end
+    end
+  end
+end

data/lib/site_maps/notification/event.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  module Notification
+    class Event
+      extend Forwardable
+
+      def_delegators :@payload, :[], :fetch, :to_h, :key?
+      alias_method :to_hash, :to_h
+
+      attr_reader :id
+
+      # Initialize a new event
+      #
+      # @param [Symbol, String] id The event identifier
+      # @param [Hash] payload
+      #
+      # @return [Event]
+      #
+      # @api private
+      def initialize(id, payload = {})
+        @id = id
+        @payload = payload
+      end
+
+      # Get or set a payload
+      #
+      # @overload
+      #   @return [Hash] payload
+      #
+      # @overload payload(data)
+      #   @param [Hash] data A new payload
+      #   @return [Event] A copy of the event with the provided payload
+      #
+      # @api public
+      def payload(data = nil)
+        if data
+          self.class.new(id, @payload.merge(data))
+        else
+          @payload
+        end
+      end
+
+      # @api private
+      def listener_method
+        @listener_method ||= Primitives::String.new("on_#{id}").underscore.to_sym
+      end
+    end
+  end
+end

data/lib/site_maps/notification/publisher.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SiteMaps::Notification
+  module Publisher
+    def self.included(klass)
+      klass.extend(ClassMethods)
+    end
+
+    # Class interface for publishers
+    #
+    # @api public
+    module ClassMethods
+      extend Forwardable
+      def_delegators :bus, :publish, :subscribed?, :unsubscribe
+
+      # Register a new event type
+      #
+      # @param [Symbol,String] event_id The event identifier
+      # @param [Hash] payload Optional default payload
+      #
+      # @return [self]
+      #
+      # @api public
+      def register_event(event_id, payload = {})
+        bus.events[event_id] = Event.new(event_id, payload)
+        self
+      end
+
+      # Publish an event with extra runtime information to the payload
+      #
+      # @param [String] event_id The event identifier
+      # @param [Hash] payload An optional payload
+      # @raise [UnregisteredEventError] if the event is not registered
+      #
+      # @api public
+      def instrument(event_id, payload = {})
+        publish_event = false # ensure block is also called on error
+        raise(UnregisteredEventError, event_id) unless bus.can_handle?(event_id)
+
+        payload[:__started_at__] = Time.now
+        yield(payload).tap { publish_event = true }
+      ensure
+        if publish_event
+          payload[:runtime] ||= Time.now - payload.delete(:__started_at__) if payload[:__started_at__]
+          bus.publish(event_id, payload)
+        end
+      end
+
+      # Subscribe to events.
+      #
+      # @param [Symbol,String,Object] object_or_event_id The event identifier or a listener object
+      # @param [Hash] filter_hash An optional event filter
+      #
+      # @raise [SiteMaps::Notification::InvalidSubscriberError] if the subscriber is not registered
+      # @return [Object] self
+      #
+      #
+      # @api public
+      def subscribe(object_or_event_id, &block)
+        if bus.can_handle?(object_or_event_id)
+          if block
+            bus.subscribe(object_or_event_id, &block)
+          else
+            bus.attach(object_or_event_id)
+          end
+
+          self
+        else
+          raise InvalidSubscriberError, object_or_event_id
+        end
+      end
+
+      def bus
+        @bus ||= Bus.new
+      end
+    end
+  end
+end

data/lib/site_maps/notification.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  module Notification
+    Error = Class.new(SiteMaps::Error)
+
+    class UnregisteredEventError < Error
+      def initialize(object_or_event_id)
+        case object_or_event_id
+        when String, Symbol
+          super("You are trying to publish an unregistered event: `#{object_or_event_id}`")
+        else
+          super("You are trying to publish an unregistered event")
+        end
+      end
+    end
+
+    class InvalidSubscriberError < Error
+      def initialize(object_or_event_id)
+        case object_or_event_id
+        when String, Symbol
+          super("you are trying to subscribe to an event: `#{object_or_event_id}` that has not been registered")
+        else
+          super("you try use subscriber object that will never be executed")
+        end
+      end
+    end
+
+    include Publisher
+
+    register_event "sitemaps.builder.finalize_urlset"
+    register_event "sitemaps.runner.before_process_execution"
+    register_event "sitemaps.runner.enqueue_process"
+    register_event "sitemaps.runner.process_execution"
+  end
+end

data/lib/site_maps/primitives/array.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module SiteMaps::Primitives
+  class Array < ::Array
+    def self.wrap(object)
+      if object.nil?
+        []
+      elsif object.respond_to?(:to_ary)
+        object.to_ary || [object]
+      else
+        [object]
+      end
+    end
+  end
+end

data/lib/site_maps/primitives/output.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+begin
+  require "rainbow"
+rescue LoadError
+end
+
+module SiteMaps
+  module Primitives
+    module Output
+      module_function
+
+      def formatted_runtime(number)
+        colorize(sprintf("%.3f ms", number), :lightgray)
+      end
+
+      def runtime_padding(number, extra = 2)
+        " " * (extra + sprintf("%.3f ms", number).size)
+      end
+
+      def colorize(text, *attributes)
+        if defined? Rainbow
+          attributes.reduce(Rainbow(text)) { |p, a| p.public_send(a) }
+        else
+          text
+        end
+      end
+
+      def print_error(message_or_error, backtrace: false, **options)
+        options[:level] ||= :error
+        message = message_or_error.to_s
+
+        print_message(message, output: :stderr, **options)
+
+        if message_or_error.is_a?(Exception) && backtrace
+          limit = backtrace.is_a?(Integer) ? backtrace : -1
+          print_backtrace(message_or_error, limit: limit, level: options[:level])
+        end
+      end
+
+      def print_backtrace(error, limit: -1, **options)
+        return unless error.respond_to?(:backtrace)
+        return if error.backtrace.nil?
+
+        error.backtrace[0..limit].each { |frame| print_error(frame, **options) }
+      end
+
+      def print_message(message, level: :info, output: $stdout, newline: true, **fields)
+        output =
+          case output
+          when :stdout, "stdout"
+            $stdout
+          when :stderr, "stderr"
+            $stderr
+          when IO, StringIO
+            output
+          else
+            raise ArgumentError, "Invalid output #{output.inspect}"
+          end
+
+        message = format(message, **fields)
+        newline ? output.puts(message) : output.print(message)
+      end
+    end
+  end
+end

data/lib/site_maps/primitives/string.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+begin
+  require "dry/inflector"
+rescue LoadError
+  # noop
+end
+
+begin
+  require "active_support/inflector"
+rescue LoadError
+  # noop
+end
+
+module SiteMaps::Primitives
+  class String < ::String
+    def classify
+      new_str = if defined?(Dry::Inflector)
+        Dry::Inflector.new.classify(self)
+      elsif defined?(ActiveSupport::Inflector)
+        ActiveSupport::Inflector.classify(self)
+      else
+        split("_").map(&:capitalize).join
+      end
+
+      self.class.new(new_str)
+    end
+
+    def underscore
+      new_str = sub(/^::/, "")
+        .gsub("::", "/")
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .tr("-", "_")
+        .tr(".", "_")
+        .gsub(/\s/, "_")
+        .gsub(/__+/, "_")
+        .downcase
+
+      self.class.new(new_str)
+    end
+  end
+end

data/lib/site_maps/process.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  Process = Concurrent::ImmutableStruct.new(:name, :location_template, :kwargs_template, :block) do
+    def location(**kwargs)
+      return unless location_template
+
+      location_template % keyword_arguments(kwargs)
+    end
+
+    def call(builder, **kwargs)
+      return unless block
+
+      block.call(builder, **keyword_arguments(kwargs))
+    end
+
+    def static?
+      !dynamic?
+    end
+
+    def dynamic?
+      kwargs_template.is_a?(Hash) && kwargs_template.any?
+    end
+
+    def keyword_arguments(given)
+      (kwargs_template || {}).merge(given || {})
+    end
+  end
+end

data/lib/site_maps/railtie.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "singleton"
+Kernel.require "rails/railtie"
+
+module SiteMaps
+  class Railtie < ::Rails::Railtie
+    initializer "site_maps.named_routes" do
+      named_route = Class.new do
+        include Singleton
+        include ::Rails.application.routes.url_helpers
+      end
+      SiteMaps::Adapters::Adapter.prepend(Module.new do
+        define_method(:route) { named_route.instance }
+      end)
+    end
+  end
+end