helios-sitemap 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 36a3e496ed0ede36fbe60b6fe276b5fd11f45b2c3a161bb101aac6202d0e46a8
+  data.tar.gz: 9213f58ce9365b6bc844c2ea4ed3fc62ca12ac60c8cd3647e341b0b0c01102f4
+SHA512:
+  metadata.gz: 5f001b4b6a6fd4fa495452272da7ed859eb19ad9cfd2409466640bc033b2b276256ad259c34df4a44314155b4bd214baa76f29b83981f5613e79ed22f2c24d95
+  data.tar.gz: 49c10dea795a509dd3fb2e3904db5c75d30584a43533054b70693d5a19b1df02dcac0c01a3cc69193377736db52e04376af9546375c2e377740d5098269f4b67

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,89 @@
+# Helios::Sitemap
+
+Sitemap generation with S3 storage and IndexNow submission for Rails. Designed for ephemeral disk systems (Heroku, Docker) where you can't persist generated sitemaps to disk.
+
+**Flow:** Generate sitemap -> Upload to S3 -> Serve from your app at `/sitemap.xml` -> Submit to IndexNow
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "helios-sitemap"
+```
+
+Then `bundle install`.
+
+## Configuration
+
+Create an initializer at `config/initializers/helios_sitemap.rb`:
+
+```ruby
+Helios::Sitemap.configure do |config|
+  config.default_host = "https://example.com"
+
+  # S3 storage (defaults to ENV vars if not set)
+  config.aws_bucket     = ENV["AWS_SITEMAP_BUCKET"]
+  config.aws_region     = ENV["AWS_REGION"]
+  config.aws_access_key_id     = ENV["AWS_ACCESS_KEY_ID"]
+  config.aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+
+  # IndexNow (defaults to ENV vars if not set)
+  config.indexnow_domain  = ENV["INDEXNOW_DOMAIN"]
+  config.indexnow_api_key = ENV["INDEXNOW_API_KEY"]
+
+  # Define what goes in your sitemap
+  config.sitemap_entries = ->(sitemap) {
+    sitemap.add "/", changefreq: "weekly", priority: 1.0
+    Post.published.find_each do |post|
+      sitemap.add "/#{post.slug}", changefreq: "weekly", priority: 0.7
+    end
+  }
+
+  # Define URLs to submit to IndexNow
+  config.indexnow_urls = -> {
+    urls = ["https://example.com/"]
+    Post.published.find_each do |post|
+      urls << "https://example.com/#{post.slug}"
+    end
+    urls
+  }
+end
+```
+
+## Routes
+
+Mount the engine in your `config/routes.rb`:
+
+```ruby
+mount Helios::Sitemap::Engine, at: "/"
+```
+
+This provides:
+- `GET /sitemap.xml` - Serves the sitemap XML from S3
+- `GET /sitemap.xml.gz` - Serves the gzipped sitemap from S3
+
+## Generating & Uploading
+
+Trigger a sitemap refresh by running the job:
+
+```ruby
+Helios::Sitemap::RefreshJob.perform_later
+```
+
+You can schedule this however you prefer (cron, recurring job, after publishing content, etc.).
+
+## Environment Variables
+
+| Variable | Description |
+|---|---|
+| `AWS_SITEMAP_BUCKET` | S3 bucket name |
+| `AWS_REGION` | AWS region |
+| `AWS_ACCESS_KEY_ID` | AWS access key |
+| `AWS_SECRET_ACCESS_KEY` | AWS secret key |
+| `INDEXNOW_DOMAIN` | Your domain for IndexNow |
+| `INDEXNOW_API_KEY` | Your IndexNow API key |
+
+## License
+
+Proprietary. All rights reserved.

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/setup"
+
+require "bundler/gem_tasks"

data/app/assets/stylesheets/helios/sitemap/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/helios/sitemap/application_controller.rb

@@ -0,0 +1,6 @@
+module Helios
+  module Sitemap
+    class ApplicationController < ActionController::Base
+    end
+  end
+end

data/app/controllers/helios/sitemap/sitemap_controller.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "aws-sdk-s3"
+require "zlib"
+
+module Helios
+  module Sitemap
+    class SitemapController < ::ActionController::Base
+      def show
+        config = Helios::Sitemap.configuration
+
+        gz_data = config.s3_client.get_object(
+          bucket: config.aws_bucket,
+          key: config.s3_object_key
+        ).body.read
+
+        if request.path.end_with?(".gz")
+          render plain: gz_data,
+                 content_type: "application/gzip",
+                 content_disposition: 'attachment; filename="sitemap.xml.gz"'
+        else
+          xml_data = Zlib::GzipReader.new(StringIO.new(gz_data)).read
+          render plain: xml_data, content_type: "application/xml"
+        end
+      rescue Aws::S3::Errors::NoSuchKey
+        head :not_found
+      end
+    end
+  end
+end

data/app/helpers/helios/sitemap/application_helper.rb

@@ -0,0 +1,6 @@
+module Helios
+  module Sitemap
+    module ApplicationHelper
+    end
+  end
+end

data/app/jobs/helios/sitemap/application_job.rb

@@ -0,0 +1,6 @@
+module Helios
+  module Sitemap
+    class ApplicationJob < ActiveJob::Base
+    end
+  end
+end

data/app/jobs/helios/sitemap/refresh_job.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "sitemap_generator"
+require "aws-sdk-s3"
+
+module Helios
+  module Sitemap
+    class RefreshJob < ApplicationJob
+      queue_as :default
+
+      def perform
+        config = Helios::Sitemap.configuration
+
+        urls = collect_urls(config)
+
+        generate_sitemap(config)
+        upload_to_s3(config) unless Rails.env.development?
+        submit_to_indexnow(urls)
+      end
+
+      private
+
+      def generate_sitemap(config)
+        Rails.logger.info("[helios-sitemap] Generating sitemap...")
+
+        SitemapGenerator::Sitemap.default_host = config.default_host
+
+        entries_proc = config.sitemap_entries
+        SitemapGenerator::Sitemap.create do
+          entries_proc&.call(self)
+        end
+      end
+
+      def upload_to_s3(config)
+        file_path = Rails.root.join("public", "sitemap.xml.gz")
+
+        transfer_manager = Aws::S3::TransferManager.new(client: config.s3_client)
+
+        transfer_manager.upload_file(
+          file_path,
+          bucket: config.aws_bucket,
+          key: config.s3_object_key,
+          content_type: "application/gzip"
+        )
+
+        Rails.logger.info("[helios-sitemap] Uploaded sitemap to s3://#{config.aws_bucket}/#{config.s3_object_key}")
+      end
+
+      def collect_urls(config)
+        return [] unless config.indexnow_urls
+
+        config.indexnow_urls.call
+      end
+
+      def submit_to_indexnow(urls)
+        return unless urls.any?
+
+        Rails.logger.info("[helios-sitemap] Submitting #{urls.count} URLs to IndexNow")
+        IndexNowService.submit_urls(urls)
+      rescue IndexNowService::IndexNowError => e
+        Rails.logger.error("[helios-sitemap] IndexNow configuration error: #{e.message}")
+      rescue StandardError => e
+        Rails.logger.error("[helios-sitemap] Unexpected error submitting to IndexNow: #{e.class} - #{e.message}")
+      end
+    end
+  end
+end

data/app/mailers/helios/sitemap/application_mailer.rb

@@ -0,0 +1,8 @@
+module Helios
+  module Sitemap
+    class ApplicationMailer < ActionMailer::Base
+      default from: "from@example.com"
+      layout "mailer"
+    end
+  end
+end

data/app/models/helios/sitemap/application_record.rb

@@ -0,0 +1,7 @@
+module Helios
+  module Sitemap
+    class ApplicationRecord < ActiveRecord::Base
+      self.abstract_class = true
+    end
+  end
+end

data/app/views/layouts/helios/sitemap/application.html.erb

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Helios sitemap</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= yield :head %>
+
+  <%= stylesheet_link_tag    "helios/sitemap/application", media: "all" %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,4 @@
+Helios::Sitemap::Engine.routes.draw do
+  get "sitemap.xml",    to: "sitemap#show"
+  get "sitemap.xml.gz", to: "sitemap#show"
+end

data/lib/helios/sitemap/configuration.rb

@@ -0,0 +1,33 @@
+module Helios
+  module Sitemap
+    class Configuration
+      attr_accessor :default_host,
+                    :aws_bucket,
+                    :aws_region,
+                    :aws_access_key_id,
+                    :aws_secret_access_key,
+                    :s3_object_key,
+                    :indexnow_domain,
+                    :indexnow_api_key,
+                    :sitemap_entries,
+                    :indexnow_urls
+
+      def initialize
+        @s3_object_key = "sitemaps/sitemap.xml.gz"
+        @aws_region = ENV["AWS_REGION"]
+        @aws_bucket = ENV["AWS_SITEMAP_BUCKET"]
+        @aws_access_key_id = ENV["AWS_ACCESS_KEY_ID"]
+        @aws_secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+        @indexnow_domain = ENV["INDEXNOW_DOMAIN"]
+        @indexnow_api_key = ENV["INDEXNOW_API_KEY"]
+      end
+
+      def s3_client
+        Aws::S3::Client.new(
+          region: aws_region,
+          credentials: Aws::Credentials.new(aws_access_key_id, aws_secret_access_key)
+        )
+      end
+    end
+  end
+end

data/lib/helios/sitemap/engine.rb

@@ -0,0 +1,7 @@
+module Helios
+  module Sitemap
+    class Engine < ::Rails::Engine
+      isolate_namespace Helios::Sitemap
+    end
+  end
+end

data/lib/helios/sitemap/index_now_service.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+
+module Helios
+  module Sitemap
+    class IndexNowService
+      ENDPOINT = "https://api.indexnow.org/IndexNow"
+
+      class IndexNowError < StandardError; end
+
+      class << self
+        def submit_urls(urls)
+          raise IndexNowError, "URLs array cannot be empty" if urls.blank?
+
+          payload = build_payload(urls)
+          make_request(payload)
+        rescue IndexNowError
+          raise
+        rescue StandardError => e
+          Rails.logger.error "IndexNow: Unexpected error - #{e.class}: #{e.message}"
+          false
+        end
+
+        private
+
+        def build_payload(urls)
+          {
+            host: domain,
+            key: api_key,
+            keyLocation: key_location_url,
+            urlList: urls
+          }
+        end
+
+        def make_request(payload)
+          uri = URI(ENDPOINT)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          http.read_timeout = 10
+          http.open_timeout = 10
+
+          request = Net::HTTP::Post.new(uri.path)
+          request["Content-Type"] = "application/json; charset=utf-8"
+          request.body = payload.to_json
+
+          response = http.request(request)
+          handle_response(response)
+        end
+
+        def handle_response(response)
+          case response.code.to_i
+          when 200, 202
+            Rails.logger.info "IndexNow: Successfully submitted URLs (#{response.code})"
+            true
+          when 400
+            Rails.logger.error "IndexNow: Bad request (400) - #{response.body}"
+            false
+          when 403
+            raise IndexNowError, "IndexNow: Forbidden (403) - Invalid or inaccessible API key. Verify key file at #{key_location_url}"
+          when 422
+            raise IndexNowError, "IndexNow: Unprocessable (422) - #{response.body}"
+          when 429
+            Rails.logger.warn "IndexNow: Rate limited (429)"
+            false
+          else
+            Rails.logger.error "IndexNow: Unexpected response (#{response.code}) - #{response.body}"
+            false
+          end
+        end
+
+        def domain
+          Helios::Sitemap.configuration.indexnow_domain || raise(IndexNowError, "indexnow_domain not configured")
+        end
+
+        def api_key
+          Helios::Sitemap.configuration.indexnow_api_key || raise(IndexNowError, "indexnow_api_key not configured")
+        end
+
+        def key_location_url
+          "https://#{domain}/#{api_key}.txt"
+        end
+      end
+    end
+  end
+end

data/lib/helios/sitemap/version.rb

@@ -0,0 +1,5 @@
+module Helios
+  module Sitemap
+    VERSION = "0.1.0"
+  end
+end

data/lib/helios/sitemap.rb

@@ -0,0 +1,18 @@
+require "helios/sitemap/version"
+require "helios/sitemap/engine"
+require "helios/sitemap/configuration"
+require "helios/sitemap/index_now_service"
+
+module Helios
+  module Sitemap
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+    end
+  end
+end

data/lib/tasks/helios/sitemap_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :helios_sitemap do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: helios-sitemap
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jason Fleetwood-Boldt
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '8.0'
+- !ruby/object:Gem::Dependency
+  name: sitemap_generator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.3'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Generate sitemaps with sitemap_generator, upload to S3 for ephemeral
+  disk systems, serve from your app at /sitemap.xml, and submit to IndexNow.
+email:
+- jason@heliosdev.shop
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/stylesheets/helios/sitemap/application.css
+- app/controllers/helios/sitemap/application_controller.rb
+- app/controllers/helios/sitemap/sitemap_controller.rb
+- app/helpers/helios/sitemap/application_helper.rb
+- app/jobs/helios/sitemap/application_job.rb
+- app/jobs/helios/sitemap/refresh_job.rb
+- app/mailers/helios/sitemap/application_mailer.rb
+- app/models/helios/sitemap/application_record.rb
+- app/views/layouts/helios/sitemap/application.html.erb
+- config/routes.rb
+- lib/helios/sitemap.rb
+- lib/helios/sitemap/configuration.rb
+- lib/helios/sitemap/engine.rb
+- lib/helios/sitemap/index_now_service.rb
+- lib/helios/sitemap/version.rb
+- lib/tasks/helios/sitemap_tasks.rake
+homepage: https://github.com/heliosdev/helios-sitemap
+licenses:
+- Nonstandard
+metadata:
+  homepage_uri: https://github.com/heliosdev/helios-sitemap
+  source_code_uri: https://github.com/heliosdev/helios-sitemap
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Sitemap generation with S3 storage and IndexNow submission for Rails
+test_files: []