site_maps 0.0.1.beta1

data/lib/site_maps/runner/event_listener.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  module Runner::EventListener
+    extend Primitives::Output
+
+    module_function
+
+    def [](event_name)
+      method_name = Primitives::String.new(event_name).underscore.to_sym
+      return unless respond_to?(:"on_#{method_name}")
+
+      method(:"on_#{method_name}")
+    end
+
+    def on_sitemaps_runner_enqueue_process(event)
+      process = event[:process]
+      kwargs = event[:kwargs]
+      location = process.location(**kwargs)
+      print_message(
+        "Enqueue process %<name>s#{" at %<location>s" if location}",
+        name: colorize(process.name, :bold),
+        location: colorize(location, :lightgray)
+      )
+      if kwargs.any?
+        print_message("└── Keyword Arguments: {%<kwargs>s}", kwargs: kwargs.map { |k, v| "#{k}: #{v.inspect}" }.join(", "))
+      end
+    end
+
+    def on_sitemaps_runner_before_process_execution(event)
+      process = event[:process]
+      kwargs = event[:kwargs]
+      location = process.location(**kwargs)
+      print_message(
+        "Executing process %<name>s#{" at %<location>s" if location}",
+        name: colorize(process.name, :bold),
+        location: colorize(location, :lightgray)
+      )
+      if kwargs.any?
+        print_message("└── Keyword Arguments: {%<kwargs>s}", kwargs: kwargs.map { |k, v| "#{k}: #{v.inspect}" }.join(", "))
+      end
+    end
+
+    def on_sitemaps_runner_process_execution(event)
+      process = event[:process]
+      kwargs = event[:kwargs]
+      location = process.location(**kwargs)
+      print_message(
+        "[%<runtime>s] Executed process %<name>s#{" at %<location>s" if location}",
+        name: colorize(process.name, :bold),
+        location: colorize(location, :lightgray),
+        runtime: formatted_runtime(event[:runtime])
+      )
+      if kwargs.any?
+        print_message("└── Keyword Arguments: {%<kwargs>s}", kwargs: kwargs.map { |k, v| "#{k}: #{v.inspect}" }.join(", "))
+      end
+    end
+
+    def on_sitemaps_builder_finalize_urlset(event)
+      links_count = event[:links_count]
+      news_count = event[:news_count]
+      url = event[:url]
+      text = +"[%<runtime>s] Finalize URLSet with "
+      text << "%<links>d links" if links_count > 0
+      text << " and " if links_count > 0 && news_count > 0
+      text << "%<news>d news" if news_count > 0
+      text << " URLs at %<url>s"
+
+      print_message(
+        text,
+        links: links_count,
+        news: news_count,
+        url: colorize(url, :lightgray),
+        runtime: formatted_runtime(event[:runtime])
+      )
+    end
+  end
+end

data/lib/site_maps/runner.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  class Runner
+    attr_reader :adapter
+
+    def initialize(adapter = SiteMaps.current_adapter, max_threads: 4)
+      @adapter = adapter.tap(&:reset!)
+      @pool = Concurrent::FixedThreadPool.new(max_threads)
+      @execution = Concurrent::Hash.new
+      @failed = Concurrent::AtomicBoolean.new(false)
+      @errors = Concurrent::Array.new
+    end
+
+    def enqueue(process_name = :default, **kwargs)
+      process_name = process_name.to_sym
+      process = @adapter.processes.fetch(process_name) do
+        raise ArgumentError, "Process :#{process_name} not found"
+      end
+      kwargs = process.keyword_arguments(kwargs)
+      SiteMaps::Notification.instrument("sitemaps.runner.enqueue_process") do |payload|
+        payload[:process] = process
+        payload[:kwargs] = kwargs
+        if process.dynamic?
+          @execution[process_name] ||= Concurrent::Array.new
+          if @execution[process_name].any? { |_, k| k == kwargs }
+            raise ArgumentError, "Process :#{process_name} already enqueued with kwargs #{kwargs}"
+          end
+          @execution[process_name] << [process, kwargs]
+        else
+          if @execution.key?(process_name)
+            raise ArgumentError, "Process :#{process_name} already defined"
+          end
+          @execution[process_name] = Concurrent::Array.new([[process, kwargs]])
+        end
+      end
+      self
+    end
+
+    def enqueue_remaining
+      @adapter.processes.each_key do |process_name|
+        next if @execution.key?(process_name)
+
+        enqueue(process_name)
+      end
+      self
+    end
+    alias_method :enqueue_all, :enqueue_remaining
+
+    def run
+      may_preload_sitemap_index
+
+      futures = []
+      @execution.each do |_process_name, items|
+        items.each do |process, kwargs|
+          SiteMaps::Notification.publish("sitemaps.runner.before_process_execution", process: process, kwargs: kwargs)
+          futures << Concurrent::Future.execute(executor: pool) do
+            wrap_process_execution(process) do
+              SiteMaps::Notification.instrument("sitemaps.runner.process_execution") do |payload|
+                payload[:process] = process
+                payload[:kwargs] = kwargs
+                builder = SiteMaps::SitemapBuilder.new(
+                  adapter: adapter,
+                  location: process.location(**kwargs)
+                )
+                process.call(builder, **kwargs)
+                builder.finalize!
+              end
+            end
+          end
+        end
+      end
+
+      futures.each(&:wait)
+      failed.false? ? finalize! : fail_with_errors!
+    ensure
+      pool.shutdown
+      pool.wait_for_termination
+      @execution.clear
+    end
+
+    private
+
+    attr_reader :pool, :failed, :errors
+
+    def finalize!
+      adapter.repo.remaining_index_links.each do |item|
+        adapter.sitemap_index.add(item)
+      end
+      unless adapter.sitemap_index.empty?
+        raw_data = adapter.sitemap_index.to_xml
+        adapter.write(adapter.config.url, raw_data, last_modified: Time.now)
+      end
+    end
+
+    def fail_with_errors!
+      return if errors.empty?
+
+      raise errors.first.last
+    end
+
+    def handle_process_error(process, error)
+      errors << [process, error]
+    end
+
+    def may_preload_sitemap_index
+      return unless preload_sitemap_index_links?
+
+      adapter.fetch_sitemap_index_links.each do |item|
+        adapter.repo.preloaded_index_links.push(item)
+      end
+    end
+
+    def wrap_process_execution(process)
+      return if failed.true?
+
+      yield
+    rescue => e
+      handle_process_error(process, e)
+      failed.make_true
+    end
+
+    def preload_sitemap_index_links?
+      return false if @execution.empty?
+
+      (@adapter.processes.keys - @execution.keys).any? || # There are processes that have not been enqueued
+        @adapter.processes.any? { |_, process| process.dynamic? } # There are dynamic processes
+    end
+
+    # def single_thread?
+    #   @pool.max_length == 1 || (
+    #     @execution.size == 1 && @execution.first.last.size == 1
+    #   )
+    # end
+  end
+end

data/lib/site_maps/sitemap_builder.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  class SitemapBuilder
+    extend Forwardable
+
+    def initialize(adapter:, location: nil)
+      @adapter = adapter
+      @url_set = SiteMaps::Builder::URLSet.new
+      @location = location
+      @mutex = Mutex.new
+    end
+
+    def add(path, params: nil, **options)
+      @mutex.synchronize do
+        link = build_link(path, params)
+        url_set.add(link, **options)
+      rescue SiteMaps::FullSitemapError
+        finalize_and_start_next_urlset!
+        url_set.add(link, **options)
+      end
+    end
+
+    def add_sitemap_index(url, lastmod: Time.now)
+      sitemap_index.add(url, lastmod: lastmod)
+    end
+
+    def finalize!
+      return if url_set.finalized?
+      return if url_set.empty?
+
+      raw_data = url_set.finalize!
+
+      SiteMaps::Notification.instrument("sitemaps.builder.finalize_urlset") do |payload|
+        payload[:links_count] = url_set.links_count
+        payload[:news_count] = url_set.news_count
+        payload[:last_modified] = url_set.last_modified
+
+        if adapter.maybe_inline_urlset? && sitemap_index.empty?
+          payload[:url] = repo.main_url
+          adapter.write(repo.main_url, raw_data, last_modified: url_set.last_modified)
+        else
+          sitemap_url = repo.generate_url(location)
+          payload[:url] = sitemap_url
+          adapter.write(sitemap_url, raw_data, last_modified: url_set.last_modified)
+          add_sitemap_index(sitemap_url, lastmod: url_set.last_modified)
+        end
+      end
+    end
+
+    protected
+
+    attr_reader :url_set, :adapter, :location
+
+    def_delegators :adapter, :sitemap_index, :config, :repo
+
+    def finalize_and_start_next_urlset!
+      raw_data = url_set.finalize!
+      SiteMaps::Notification.instrument("sitemaps.builder.finalize_urlset") do |payload|
+        sitemap_url = repo.generate_url(location)
+        payload[:url] = sitemap_url
+        payload[:links_count] = url_set.links_count
+        payload[:news_count] = url_set.news_count
+        payload[:last_modified] = url_set.last_modified
+        adapter.write(sitemap_url, raw_data, last_modified: url_set.last_modified)
+        add_sitemap_index(sitemap_url, lastmod: url_set.last_modified)
+      end
+      @url_set = SiteMaps::Builder::URLSet.new
+    end
+
+    def build_link(path, params)
+      SiteMaps::Builder::Link.new(config.base_uri, path, params)
+    end
+  end
+end

data/lib/site_maps/sitemap_reader.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "open-uri"
+
+module SiteMaps
+  class SitemapReader
+    Error = Class.new(SiteMaps::Error)
+    FileNotFound = Class.new(Error)
+    MalformedFile = Class.new(Error)
+
+    def initialize(location)
+      @location = Pathname.new(location)
+    end
+
+    def read
+      if compressed?
+        Zlib::GzipReader.new(read_file).read
+      else
+        read_file.read
+      end
+    rescue Zlib::GzipFile::Error => _e
+      raise MalformedFile.new("The file #{@location} is not a valid Gzip file")
+    end
+
+    def to_doc
+      @doc ||= begin
+        require "nokogiri"
+        Nokogiri::XML(read)
+      end
+    rescue LoadError
+      raise SiteMaps::Error, "Nokogiri is required to parse the XML file"
+    end
+
+    protected
+
+    def read_file
+      if remote?
+        ::URI.parse(@location.to_s).open
+      else
+        ::File.open(@location, "r")
+      end
+    rescue Errno::ENOENT
+      raise FileNotFound.new("The file #{@location} does not exist")
+    rescue OpenURI::HTTPError
+      raise FileNotFound.new("The file #{@location} could not be opened")
+    end
+
+    def compressed?
+      @location.extname == ".gz"
+    end
+
+    def remote?
+      %r{^https?://}.match?(@location.to_s)
+    end
+  end
+end

data/lib/site_maps/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SiteMaps
+  VERSION = "0.0.1.beta1"
+end

data/lib/site_maps.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require_relative "site_maps/version"
+
+require "builder"
+require "concurrent-ruby"
+require "date"
+require "fileutils"
+require "forwardable"
+require "rack/utils"
+require "stringio"
+require "time"
+require "uri"
+require "zeitwerk"
+require "zlib"
+
+loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false)
+loader.ignore("#{__dir__}/site-maps.rb")
+loader.ignore("#{__dir__}/site_maps/tasks.rb")
+loader.ignore("#{__dir__}/site_maps/cli.rb")
+loader.inflector.inflect "cli" => "CLI"
+loader.inflector.inflect "dsl" => "DSL"
+loader.inflector.inflect "url_set" => "URLSet"
+loader.inflector.inflect "url" => "URL"
+loader.inflector.inflect "xml" => "XML"
+loader.log! if ENV["DEBUG_ZEITWERK"]
+loader.setup
+
+module SiteMaps
+  MAX_LENGTH = {
+    links: 50_000,
+    images: 1_000,
+    news: 1_000
+  }
+  MAX_FILESIZE = 50_000_000 # bytes
+  DEFAULT_LOGGER = Logger.new($stdout)
+
+  Error = Class.new(StandardError)
+  AdapterNotFound = Class.new(Error)
+  AdapterNotSetError = Class.new(Error)
+  FileNotFoundError = Class.new(Error)
+  FullSitemapError = Class.new(Error)
+  ConfigurationError = Class.new(Error)
+
+  class << self
+    attr_reader :current_adapter
+    attr_writer :logger
+
+    # @param adapter [Class, String, Symbol] The name of the adapter to use
+    # @param options [Hash] Options to pass to the adapter. Note that these are adapter-specific
+    # @param block [Proc] A block to pass to the adapter
+    # @return [Object] An instance of the adapter
+    def use(adapter, **options, &block)
+      adapter_class = if adapter.is_a?(Class) # && adapter < Adapters::Adapter
+        adapter
+      else
+        const_name = Primitives::String.new(adapter.to_s).classify
+        begin
+          Adapters.const_get(const_name)
+        rescue NameError
+          raise AdapterNotFound, "Adapter #{adapter.inspect} not found"
+        end
+      end
+      @current_adapter = adapter_class.new(**options, &block)
+    end
+
+    def config
+      @config ||= Configuration.new
+      yield(@config) if block_given?
+      @config
+    end
+    alias_method :configure, :config
+
+    # Load and prepare a runner with the current adapter
+    # Note that it won't start running until you call `#run` on the runner
+    #
+    # Example:
+    #   SiteMaps.generate(config_file: "config/site_maps.rb", max_threads: 10)
+    #     .enqueue_all
+    #     .run
+    #
+    # You may also enqueue processes manually, specially those that are dynamic
+    #
+    # Example:
+    #   SiteMaps.generate(config_file: "config/site_maps.rb", max_threads: 10)
+    #     .enqueue(:monthly, year: 2020, month: 1)
+    #     .enqueue(:monthly, year: 2020, month: 2)
+    #     .enqueue_remaining # Enqueue all other non-enqueued processes
+    #     .run
+    #
+    # @param config_file [String] The path to a configuration file
+    # @param options [Hash] Options to pass to the runner
+    # @return [Runner] An instance of the runner
+    def generate(config_file: nil, **options)
+      if config_file
+        @current_adapter = nil
+        load(config_file)
+      end
+      raise AdapterNotSetError, "No adapter set. Use SiteMaps.use to set an adapter" unless current_adapter
+
+      Runner.new(current_adapter, **options)
+    end
+
+    def logger
+      @logger ||= DEFAULT_LOGGER
+    end
+  end
+end
+
+if defined?(::Rails)
+  require_relative "site_maps/railtie"
+end

data/site_maps.gemspec

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "lib/site_maps/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "site_maps"
+  spec.version = SiteMaps::VERSION
+  spec.authors = ["Marcos G. Zimmermann"]
+  spec.email = ["mgzmaster@gmail.com"]
+
+  spec.summary = <<~SUMMARY
+    SiteMaps simplifies the generation of sitemaps for ruby or rails applications.
+  SUMMARY
+  spec.description = <<~DESCRIPTION
+    SiteMaps is gem that provides a simple way to generate static sitemaps for your ruby or rails applications.
+  DESCRIPTION
+
+  spec.homepage = "https://github.com/marcosgz/site_maps"
+  spec.license = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
+
+  raise "RubyGems 2.0 or newer is required to protect against public gem pushes." unless spec.respond_to?(:metadata)
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["bug_tracker_uri"] = "https://github.com/marcosgz/site_maps/issues"
+  spec.metadata["documentation_uri"] = "https://github.com/marcosgz/site_maps"
+  spec.metadata["source_code_uri"] = "https://github.com/marcosgz/site_maps"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.bindir = "exec"
+  spec.executables = spec.files.grep(%r{^exec/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rack", ">= 2.0"
+  spec.add_dependency "zeitwerk", ">= 0"
+  spec.add_dependency "builder", "~> 3.0"
+  spec.add_dependency "concurrent-ruby", ">= 1.1"
+  spec.add_dependency "thor", ">= 0.0.0"
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: site_maps
+version: !ruby/object:Gem::Version
+  version: 0.0.1.beta1
+platform: ruby
+authors:
+- Marcos G. Zimmermann
+autorequire:
+bindir: exec
+cert_chain: []
+date: 2024-11-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: builder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.0
+description: 'SiteMaps is gem that provides a simple way to generate static sitemaps
+  for your ruby or rails applications.
+
+  '
+email:
+- mgzmaster@gmail.com
+executables:
+- site_maps
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/main.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".tool-versions"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exec/site_maps
+- lib/site-maps.rb
+- lib/site_maps.rb
+- lib/site_maps/adapters/adapter.rb
+- lib/site_maps/adapters/aws_sdk.rb
+- lib/site_maps/adapters/aws_sdk/config.rb
+- lib/site_maps/adapters/aws_sdk/location.rb
+- lib/site_maps/adapters/aws_sdk/storage.rb
+- lib/site_maps/adapters/file_system.rb
+- lib/site_maps/adapters/file_system/config.rb
+- lib/site_maps/adapters/file_system/location.rb
+- lib/site_maps/adapters/file_system/storage.rb
+- lib/site_maps/adapters/noop.rb
+- lib/site_maps/atomic_repository.rb
+- lib/site_maps/builder/link.rb
+- lib/site_maps/builder/normalizer.rb
+- lib/site_maps/builder/sitemap_index.rb
+- lib/site_maps/builder/sitemap_index/item.rb
+- lib/site_maps/builder/url.rb
+- lib/site_maps/builder/url_set.rb
+- lib/site_maps/cli.rb
+- lib/site_maps/configuration.rb
+- lib/site_maps/incremental_location.rb
+- lib/site_maps/notification.rb
+- lib/site_maps/notification/bus.rb
+- lib/site_maps/notification/event.rb
+- lib/site_maps/notification/publisher.rb
+- lib/site_maps/primitives/array.rb
+- lib/site_maps/primitives/output.rb
+- lib/site_maps/primitives/string.rb
+- lib/site_maps/process.rb
+- lib/site_maps/railtie.rb
+- lib/site_maps/runner.rb
+- lib/site_maps/runner/event_listener.rb
+- lib/site_maps/sitemap_builder.rb
+- lib/site_maps/sitemap_reader.rb
+- lib/site_maps/version.rb
+- site_maps.gemspec
+homepage: https://github.com/marcosgz/site_maps
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/marcosgz/site_maps
+  bug_tracker_uri: https://github.com/marcosgz/site_maps/issues
+  documentation_uri: https://github.com/marcosgz/site_maps
+  source_code_uri: https://github.com/marcosgz/site_maps
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.21
+signing_key:
+specification_version: 4
+summary: SiteMaps simplifies the generation of sitemaps for ruby or rails applications.
+test_files: []