site_maps 0.0.1.beta1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 58b8a07d1b6a6b8774a2ee2d1dd0e3ae9ebea70095ebbf5b3c0abe4edfdece6b
+  data.tar.gz: 472ea2dcc6760eb0f072f9ffb30b8f31fbf4fcbc25041802f44daed83c81f27c
+SHA512:
+  metadata.gz: f86928a65fb91dc0f655bf45d87c3b2eedcb1fdeb08a99f509346ec9e7d8388220386322069ba6a9ecc9fb7abab667e5683ea1be42816a8dca83462251f58f27
+  data.tar.gz: 4fafbd0e7da884694f13bf4cc6ff47db28ad8e6b04fcc0680eb671d45fbddafbec1f3d0f6420f5d6190ee439b3f9ecf74cc1307e593c8d11b97e230255788e2f

data/.github/workflows/main.yml

@@ -0,0 +1,45 @@
+name: Build
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  rubocop:
+    name: "Rubocop"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.0"
+          bundler-cache: true
+      - name: Run rubocop
+        run: |
+          bundle exec rubocop --parallel
+  tests:
+    name: Tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - "3.0"
+          - "3.1"
+          - "3.2"
+          - "3.3"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Ruby and install gems
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+      - name: Run tests
+        run: bundle exec rspec
+

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.env
+/.rspec_status
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+/spec/dummy/db/*.sqlite3
+/spec/dummy/db/*.sqlite3-*
+/spec/dummy/log/*.log
+/spec/dummy/storage/
+/spec/dummy/tmp/

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,36 @@
+inherit_mode:
+  merge:
+    - Exclude
+
+require:
+  - rubocop-performance
+  - rubocop-rspec
+  - standard/cop/block_single_line_braces
+
+inherit_gem:
+  standard: config/base.yml
+
+AllCops:
+  TargetRubyVersion: 2.5
+  SuggestExtensions: false
+  Exclude:
+    - "db/**/*"
+    - "tmp/**/*"
+    - "vendor/**/*"
+    - "spec/support/hooks/notification.rb" # I'll refactor this rspec matcher to allow use things like an_instance_of
+  NewCops: enable
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/MessageSpies:
+  Enabled: false
+
+RSpec/StubbedMock:
+  Enabled: false

data/.tool-versions

@@ -0,0 +1 @@
+ruby 3.3.5

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 0.0.1.beta1 - 2024-11-07
+The first release of the gem

data/Gemfile

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "aws-sdk-s3"
+gem "dotenv"
+gem "nokogiri"
+gem "pry"
+gem "rspec"
+gem "rubocop-performance"
+gem "rubocop-rspec"
+gem "rubocop"
+gem "standard"
+gem "timecop"
+gem "webmock"
+
+# Specify your gem's dependencies in site_maps.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,134 @@
+PATH
+  remote: .
+  specs:
+    site_maps (0.0.1.beta1)
+      builder (~> 3.0)
+      concurrent-ruby (>= 1.1)
+      rack (>= 2.0)
+      thor
+      zeitwerk
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.7)
+      public_suffix (>= 2.0.2, < 7.0)
+    ast (2.4.2)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.1003.0)
+    aws-sdk-core (3.212.0)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.992.0)
+      aws-sigv4 (~> 1.9)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-kms (1.95.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sigv4 (~> 1.5)
+    aws-sdk-s3 (1.170.0)
+      aws-sdk-core (~> 3, >= 3.210.0)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.10.1)
+      aws-eventstream (~> 1, >= 1.0.2)
+    bigdecimal (3.1.8)
+    builder (3.3.0)
+    coderay (1.1.3)
+    concurrent-ruby (1.3.4)
+    crack (1.0.0)
+      bigdecimal
+      rexml
+    diff-lcs (1.5.1)
+    dotenv (2.8.1)
+    hashdiff (1.1.1)
+    jmespath (1.6.2)
+    json (2.8.1)
+    language_server-protocol (3.17.0.3)
+    lint_roller (1.1.0)
+    method_source (1.1.0)
+    nokogiri (1.16.7-x86_64-linux)
+      racc (~> 1.4)
+    parallel (1.26.3)
+    parser (3.3.6.0)
+      ast (~> 2.4.1)
+      racc
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (5.1.1)
+    racc (1.8.1)
+    rack (3.1.8)
+    rainbow (3.1.1)
+    regexp_parser (2.9.2)
+    rexml (3.3.9)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.1)
+    rubocop (1.64.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.34.1)
+      parser (>= 3.3.1.0)
+    rubocop-performance (1.21.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+    rubocop-rspec (3.2.0)
+      rubocop (~> 1.61)
+    ruby-progressbar (1.13.0)
+    standard (1.37.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.64.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.4)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.4.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.21.0)
+    thor (1.3.2)
+    timecop (0.9.10)
+    unicode-display_width (2.6.0)
+    webmock (3.24.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    zeitwerk (2.6.18)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  aws-sdk-s3
+  dotenv
+  nokogiri
+  pry
+  rspec
+  rubocop
+  rubocop-performance
+  rubocop-rspec
+  site_maps!
+  standard
+  timecop
+  webmock
+
+BUNDLED WITH
+   2.3.22

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Marcos G. Zimmermann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,186 @@
+# SiteMaps
+
+SiteMaps is a gem that helps you to generate sitemaps for your Rails application.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'site_maps'
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+Or install it yourself as:
+
+```bash
+gem install site_maps
+```
+
+## Usage
+
+Create a configuration file where you will define the sitemap logic. You can use the following DSL to define the sitemap generation. Below is the minimum configuration required to generate a sitemap:
+
+```ruby
+# config/sitemap.rb
+SiteMaps.use(:file_system) do
+  configure do |config|
+    config.url = "https://example.com/sitemaps/sitemap.xml.gz" # Location of main sitemap index file
+    config.directory = "/home/www/public"
+  end
+  process do |s|
+    s.add('/', priority: 1.0, changefreq: "daily")
+    s.add('/about', priority: 0.9, changefreq: "weekly")
+  end
+end
+```
+
+After creating the configuration file, you can run the following command to generate the sitemap:
+
+```bash
+SiteMaps.generate(config_file: "config/sitemap.rb")
+  .enqueue_all
+  .run
+```
+
+Sitemap links are defined in the `process` block because the gem is designed to generate sitemaps for large websites in parallel. The `process` block will be executed in a separate thread for each process, which will improve the performance of the sitemap generation. Each process can have a unique name and a unique sitemap file location. By omitting the name and the file location, the process will use the `:default` value.
+
+Bellows is an example of a configuration file with multiple processes:
+
+```ruby
+# config/sitemap.rb
+SiteMaps.use(:file_system) do
+  configure do |config|
+    config.url = "https://example.com/sitemaps/sitemap.xml.gz" # Location of main sitemap index file
+    config.directory = "/home/www/public"
+  end
+  process do |s|
+    s.add('/', priority: 1.0, changefreq: "daily")
+    s.add('/about', priority: 0.9, changefreq: "weekly")
+  end
+  process :categories, "categories/sitemap.xml" do |s|
+    Category.find_each do |category|
+      s.add(category_path(category), priority: 0.7)
+    end
+  end
+  process :posts, "posts/%{year}-%{month}/sitemap.xml", year: Date.today.year, month: Date.today.month do |s, year, month|
+    Post.where(year: year, month: month).find_each do |post|
+      s.add(post_path(post), priority: 0.8)
+    end
+  end
+end
+```
+
+The `process` block can receive a name and a file location as arguments. The file location can contain placeholders that will be replaced by the values passed to the process block. The `process` block can receive a hash with the values that will be used to replace the placeholders in the file location. It will allow you to incrementally generate sitemaps by year and month, for example.
+
+```ruby
+SiteMaps.generate(config_file: "config/sitemap.rb")
+  .enqueue(:posts, year: 2021, month: 1)
+  .enqueue(:posts, year: 2021, month: 2)
+  .enqueue_remaining # Enqueue all remaining processes (default and categories)
+  .run
+```
+
+If you are using Rails, you may want to add routes to the sitemap builder. You can use the `include_module` adapter method.
+
+```ruby
+# config/sitemap.rb
+SiteMaps.use(:file_system) do
+  include_module Rails.application.routes.url_helpers # It's the same of `extend Rails.application.routes.url_helpers`
+
+  configure do |config|
+    config.url = "https://example.com/sitemaps/sitemap.xml.gz" # Location of main sitemap index file
+    config.directory = "/home/www/public"
+  end
+  process do |s|
+    s.add(root_path, priority: 1.0, changefreq: "daily")
+    s.add(about_path, priority: 0.9, changefreq: "weekly")
+  end
+end
+```
+
+### AWS S3
+
+You can use the AWS S3 adapter to store the sitemaps in an S3 bucket. The configuration is similar to the file system adapter, but you need to provide the AWS SDK options.
+
+```ruby
+aws_sdk_options = {
+  bucket: "my-bucket",
+  region: "us-east-1",
+  aws_access_key: ENV["AWS_ACCESS_KEY_ID"],
+  aws_secret_key: ENV["AWS_SECRET_ACCESS_KEY"],
+  # Optional parameters (default values)
+  acl: "public-read",
+  cache_control: "private, max-age=0, no-cache",
+}
+
+SiteMaps.use(:aws_sdk, **aws_sdk_options) do
+  configure do |config|
+    config.url = "https://my-bucket.s3.amazonaws.com/sitemaps/sitemap.xml.gz"
+  end
+  process do |s|
+    s.add('/', priority: 1.0, changefreq: "daily")
+    s.add('/about', priority: 0.9, changefreq: "weekly")
+  end
+end
+```
+
+## CLI
+
+You can use the CLI to generate the sitemap. The CLI will load the configuration file and run the sitemap generation.
+
+```bash
+bundle exec site_maps generate --config-file config/sitemap.rb
+```
+
+To enqueue dynamic processes, you can pass the process name with the context values.
+
+```bash
+bundle exec site_maps generate monthly_posts --config-file config/sitemap.rb --context=year:2021,month:1
+```
+
+## Notification
+
+You can subscribe to the internal events to receive notifications about the sitemap generation. The following events are available:
+
+* `sitemaps.runner.enqueue_process` - Triggered when a process is enqueued.
+* `sitemaps.runner.before_process_execution` - Triggered before a process starts execution
+* `sitemaps.runner.process_execution` - Triggered when a process finishes execution.
+* `sitemaps.builder.finalize_urlset` - Triggered when the sitemap builder finishes the URL set.
+
+You can subscribe to the events using the following code:
+
+```ruby
+SiteMaps::Notification.subscribe("sitemaps.runner.enqueue_process") do |event|
+  puts "Enqueueing process #{event.payload[:name]}"
+end
+```
+
+We have the standard event handler `SiteMaps::Runner::EventListener` that will print the events to the standard output. You can use it to view the progress of the sitemap generation.
+
+```ruby
+SiteMaps::Notification.subscribe(SiteMaps::Runner::EventListener)
+SiteMaps.generate(config_file: "config/sitemap.rb")
+  .enqueue_all
+  .run
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/marcosgz/site_maps.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "dotenv/load"
+require "pry"
+require "site_maps"
+
+SiteMaps.configure do |config|
+  config.url = "https://example.com/site_maps/sitemap.xml"
+  config.directory = "/tmp/site_maps"
+end
+
+Pry.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+find gemfiles -type f \( -iname "*.gemfile" ! -iname "*.lock" \) -exec bundle install --gemfile {} \;

data/exec/site_maps

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+
+require "site_maps"
+require "site_maps/cli"
+
+SiteMaps::CLI.start

data/lib/site-maps.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "site_maps"

data/lib/site_maps/adapters/adapter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module SiteMaps::Adapters
+  class Adapter
+    extend Forwardable
+
+    class << self
+      def config_class
+        return SiteMaps::Configuration unless defined?(self::Config)
+
+        self::Config
+      end
+    end
+
+    def_delegators :config, :fetch_sitemap_index_links
+    attr_reader :sitemap_index, :processes
+
+    def initialize(**options, &block)
+      @config = SiteMaps.config.becomes(self.class.config_class, **options)
+      @processes = Concurrent::Hash.new
+      reset!
+      instance_exec(&block) if block
+    end
+
+    # @abstract
+    # @param [String] url The remote URL to write to
+    # @param [String] raw_data The raw data to write
+    # @return [void]
+    # @raise [SiteMaps::Error] if the write operation fails
+    def write(_url, _raw_data, **_kwargs)
+      raise NotImplementedError
+    end
+
+    # @abstract
+    # @param [String] url The remote URL to read from
+    # @return [Array<String, Hash>] The raw data and metadata
+    # @raise [SiteMaps::FileNotFoundError] if the file does not exist
+    def read(_url)
+      raise NotImplementedError
+    end
+
+    # @abstract
+    # @param [String] url The remote URL to delete
+    # @return [void]
+    # @raise [SiteMaps::FileNotFoundError] if the file does not exist
+    def delete(_url)
+      raise NotImplementedError
+    end
+
+    def config
+      yield(@config) if block_given?
+      @config
+    end
+    alias_method :configure, :config
+
+    def process(name = :default, location = nil, **kwargs, &block)
+      name = name.to_sym
+      raise ArgumentError, "Process #{name} already defined" if @processes.key?(name)
+
+      @processes[name] = SiteMaps::Process.new(name, location, kwargs, block)
+    end
+
+    def maybe_inline_urlset?
+      @processes.size == 1 && @processes.first.last.static?
+    end
+
+    def repo
+      @repo ||= SiteMaps::AtomicRepository.new(config.url)
+    end
+
+    def include_module(mod)
+      extend(mod)
+    end
+
+    def reset!
+      @sitemap_index = SiteMaps::Builder::SitemapIndex.new
+      @repo = nil
+    end
+  end
+end

data/lib/site_maps/adapters/aws_sdk/config.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+class SiteMaps::Adapters::AwsSdk::Config < SiteMaps::Configuration
+  attribute :access_key_id, default: -> { ENV["AWS_ACCESS_KEY_ID"] }
+  attribute :secret_access_key, default: -> { ENV["AWS_SECRET_ACCESS_KEY"] }
+  attribute :region, default: -> { ENV.fetch("AWS_REGION", "us-east-1") }
+  attribute :bucket, default: -> { ENV["AWS_BUCKET"] }
+  attribute :acl, default: "public-read"
+  attribute :cache_control, default: "private, max-age=0, no-cache"
+
+  attr_reader :aws_extra_options
+
+  def initialize(**options)
+    defined_attrs = options.slice(*self.class.attributes.keys)
+    super(**defined_attrs)
+
+    @aws_extra_options = options.reject { |k, v| defined_attrs.key?(k) }
+  end
+
+  def s3_bucket
+    s3_resource.bucket(bucket)
+  end
+
+  def s3_resource
+    @s3_resource ||= begin
+      require "aws-sdk-s3"
+
+      ::Aws::S3::Resource.new(s3_resource_options)
+    end
+  end
+
+  def inspect
+    "#<#{self.class}:#{object_id} access_key_id=#{access_key_id.inspect} region=#{region.inspect} bucket=#{bucket.inspect}>"
+  end
+  alias_method :to_s, :inspect
+
+  private
+
+  def s3_resource_options
+    options = aws_extra_options.dup
+    options[:region] = region if region?
+    if access_key_id? && secret_access_key?
+      options[:credentials] = ::Aws::Credentials.new(
+        access_key_id,
+        secret_access_key
+      )
+    end
+
+    options
+  end
+end

data/lib/site_maps/adapters/aws_sdk/location.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class SiteMaps::Adapters::AwsSdk::Location < SiteMaps::Adapters::FileSystem::Location
+  ROOT_RE = %r{^/}
+
+  def remote_path
+    make_relative(uri.path)
+  end
+end