crawlscope 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c7c5616af9b17e185b54274dfa07997d712f93ae37285d1cc9dbe190d289882
-  data.tar.gz: a443e0cfdce5c3a1a08bdcc0b53916a6e60631aad5c3ef5c9cc5377e70558383
+  metadata.gz: 18717d5f025a5e0dc10b3f5a231b52d788a7ad09fe017b803815dc1cf22b7f11
+  data.tar.gz: 25f80cac563922d3c34d069c34b6c5521064fb4a74dd98cc54f6bb04e6de283a
 SHA512:
-  metadata.gz: f00dbb412e2372bbe757b7fdf84dadc5e7cb05f82f2e254a08a5b8a199020b3cbdceff460ecf37f53a93d9617c769be1673120599edade1f8e9881c5a2a56de7
-  data.tar.gz: 462cce5554b2872955c76855bf60a77e93948c1162219ec129b7d3a5e511a3726b73157d33db8a4b9c3a64564e2c69306138b9c0ff62600b67570e1d6fee0403
+  metadata.gz: 322dacf831bfa19dd16a6ebf3917e8d8bcaab462b440468945e4e9e07768a42955afe7f418b217594f9654c22490d32124f6fdb95004a20f23508f76bca6d7db
+  data.tar.gz: b301ad0f2c7c92dd3dc436f7019c62d7958bdd402eb8a887626d938a9bfd085d8d21bdab76a5243f3e2d9684d94ffce554e5108db26995d67b63140fcadbb9e2

data/CHANGELOG.md

@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.2] - 2026-07-23
+
+
+### Fixed
+
+- preserve host configuration
+
+
+
+## [0.7.1] - 2026-07-21
+
+
+### Added
+
+- add lazy install generator
+
+
+
 ## [0.7.0] - 2026-07-21
 
 

data/README.md

@@ -142,24 +142,23 @@ puts result.issues.to_a.map(&:message)
 - `pages`: fetched page snapshots
 - `issues`: structured issues with `code`, `severity`, `category`, `url`, and `message`
 
-`result.ok?` returns `false` if any error, warning, or notice is present.
+`result.ok?` returns `false` when an error is present. Warnings and notices
+remain available through `result.issues` without making the result fail.
 
 ## Rails Usage
 
-For an app that uses Crawlscope through rake tasks, load the task entrypoint
-only when a Crawlscope task is requested. In the application's `Rakefile`,
-before `require_relative "config/application"`:
+Run the install generator after adding the gem:
 
-```ruby
-list_tasks = Rake.application.options.show_tasks || ARGV.any? { |argument| ["-T", "--tasks"].include?(argument) }
-require "crawlscope/tasks" if list_tasks || ARGV.any? { |argument| argument.start_with?("crawlscope:") }
+```bash
+bin/rails generate crawlscope:install
 ```
 
-This preserves `bin/rails --tasks` and the `crawlscope:*` tasks without loading
-Crawlscope during normal Rails boot. Runtime callers should explicitly
-`require "crawlscope"` immediately before configuring or using the gem.
+The generator creates `config/initializers/crawlscope.rb` with an idempotent
+`CrawlscopeConfiguration.apply` loader and adds conditional `crawlscope/tasks`
+loading to the application's `Rakefile`. This preserves `bin/rails --tasks` and
+the `crawlscope:*` tasks without loading Crawlscope during normal Rails boot.
 
-In an initializer:
+Customize the `Crawlscope.configure` block inside the generated initializer:
 
 ```ruby
 Crawlscope.configure do |config|
@@ -170,6 +169,20 @@ Crawlscope.configure do |config|
 end
 ```
 
+Runtime callers must apply the generated configuration before using
+Crawlscope:
+
+```ruby
+CrawlscopeConfiguration.apply
+Crawlscope.configuration.audit
+```
+
+Rake tasks apply it automatically because `crawlscope/tasks` loads the gem
+before Rails evaluates the initializer. Each Rake entry point passes the shared
+`Crawlscope.configuration` object to the CLI, so configured base URLs, sitemap
+paths, registries, and runtime settings are preserved unless a task argument or
+environment override replaces them.
+
 Then run:
 
 ```bash
@@ -211,7 +224,12 @@ bundle exec rake crawlscope:validate:ldjson URL=https://example.com/article
 bundle exec rake 'crawlscope:validate:ldjson[https://example.com/article]'
 ```
 
-`crawlscope:validate` runs all default sitemap rules: indexability, metadata, structured data, uniqueness, content quality, and links. `URL` is the site base. Without `SITEMAP`, Crawlscope uses `/sitemap.xml`. With `SITEMAP`, Crawlscope uses `URL` as the site base and validates URLs from that sitemap. `SITEMAP` may be a full URL or a local file path.
+`crawlscope:validate` runs all default sitemap rules: indexability, metadata,
+structured data, uniqueness, content quality, and links. `URL` is the site
+base. Without `SITEMAP`, Crawlscope uses the configured sitemap path, then
+falls back to `/sitemap.xml`. With `SITEMAP`, Crawlscope uses `URL` as the site
+base and validates URLs from that sitemap. `SITEMAP` may be a full URL or a
+local file path.
 
 Plain `rake` does not pass `--url` style flags to tasks. Use `URL=...` or the
 task-argument form above instead.

data/lib/crawlscope/rake_tasks.rb

@@ -16,8 +16,8 @@ module Crawlscope
       validate(url: url, sitemap_path: sitemap_path, rule_names: rule)
     end
 
-    def run(command, argv: [])
-      status = Cli.start([command, *argv], out: $stdout, err: $stderr)
+    def run(command, argv: [], configuration: Crawlscope.configuration)
+      status = Cli.start([command, *argv], out: $stdout, err: $stderr, configuration: configuration)
       exit(status) unless status.zero?
     end
 

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.7.0"
+  VERSION = "0.7.2"
 end

data/lib/crawlscope.rb

@@ -27,6 +27,7 @@ module Crawlscope
 end
 
 Crawlscope.instance_variable_set(:@loader, Zeitwerk::Loader.for_gem)
+Crawlscope.loader.ignore("#{__dir__}/generators")
 Crawlscope.loader.ignore("#{__dir__}/tasks")
 Crawlscope.loader.ignore("#{__dir__}/crawlscope/railtie.rb")
 Crawlscope.loader.ignore("#{__dir__}/crawlscope/tasks.rb")

data/lib/generators/crawlscope/install_generator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Crawlscope
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Create a lazy-safe Crawlscope initializer and task loader"
+
+      def create_initializer
+        template "initializer.rb.tt", "config/initializers/crawlscope.rb"
+      end
+
+      def configure_rakefile
+        rakefile = File.join(destination_root, "Rakefile")
+        return unless File.exist?(rakefile)
+        return if File.read(rakefile).include?('require "crawlscope/tasks"')
+
+        inject_into_file "Rakefile", rakefile_setup, before: /^require_relative ["']config\/application["']/
+      end
+
+      private
+
+      def rakefile_setup
+        <<~RUBY
+          crawlscope_tasks_requested = Rake.application.options.show_tasks ||
+            ARGV.any? { |argument| ["-T", "--tasks"].include?(argument) }
+          require "crawlscope/tasks" if crawlscope_tasks_requested ||
+            ARGV.any? { |argument| argument.start_with?("crawlscope:") }
+
+        RUBY
+      end
+    end
+  end
+end

data/lib/generators/crawlscope/templates/initializer.rb.tt

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module CrawlscopeConfiguration
+  MUTEX = Mutex.new
+
+  def self.apply
+    return if @applied
+
+    MUTEX.synchronize do
+      return if @applied
+
+      require "crawlscope"
+
+      Crawlscope.configure do |config|
+        config.base_url = -> { ENV.fetch("CRAWLSCOPE_BASE_URL", "http://localhost:3000") }
+        config.sitemap_path = lambda {
+          ENV.fetch("SITEMAP", Rails.public_path.join("sitemap.xml").to_s)
+        }
+        config.site_name = ENV.fetch("CRAWLSCOPE_SITE_NAME", "Application")
+      end
+
+      @applied = true
+    end
+  end
+end
+
+CrawlscopeConfiguration.apply if defined?(Crawlscope)

data/test/crawlscope/install_generator_test.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "generators/crawlscope/install_generator"
+require "open3"
+require "rbconfig"
+
+class CrawlscopeInstallGeneratorTest < Minitest::Test
+  def test_generates_lazy_initializer_and_task_loader
+    Dir.mktmpdir do |destination|
+      File.write(File.join(destination, "Rakefile"), <<~RUBY)
+        require_relative "config/application"
+        Rails.application.load_tasks
+      RUBY
+
+      Crawlscope::Generators::InstallGenerator.start([], destination_root: destination)
+
+      initializer = File.join(destination, "config/initializers/crawlscope.rb")
+      rakefile = File.read(File.join(destination, "Rakefile"))
+
+      assert File.exist?(initializer)
+      assert_includes File.read(initializer), "module CrawlscopeConfiguration"
+      assert_includes File.read(initializer), "CrawlscopeConfiguration.apply if defined?(Crawlscope)"
+      assert_includes rakefile, 'require "crawlscope/tasks"'
+      assert_operator rakefile.index('require "crawlscope/tasks"'), :<, rakefile.index('require_relative "config/application"')
+
+      _stdout, stderr, status = Open3.capture3(RbConfig.ruby, initializer)
+      assert status.success?, stderr
+
+      script = <<~RUBY
+        require "pathname"
+
+        module Rails
+          def self.public_path
+            Pathname.new("public")
+          end
+        end
+
+        require "crawlscope"
+        load ARGV.fetch(0)
+
+        configuration = Crawlscope.configuration
+        CrawlscopeConfiguration.apply
+        abort "configuration replaced" unless Crawlscope.configuration.equal?(configuration)
+        abort "base URL missing" unless configuration.base_url == "http://localhost:3000"
+        abort "sitemap missing" unless configuration.sitemap_path == "public/sitemap.xml"
+      RUBY
+      _stdout, stderr, status = Open3.capture3(
+        RbConfig.ruby,
+        "-I#{File.expand_path("../../lib", __dir__)}",
+        "-e",
+        script,
+        initializer
+      )
+      assert status.success?, stderr
+    end
+  end
+end

data/test/crawlscope/rake_tasks_test.rb

@@ -3,6 +3,8 @@
 require "test_helper"
 
 class CrawlscopeRakeTasksTest < Minitest::Test
+  Rule = Data.define(:code)
+
   def setup
     @original_start = Crawlscope::Cli.method(:start)
   end
@@ -13,6 +15,7 @@ class CrawlscopeRakeTasksTest < Minitest::Test
     singleton_class.define_method(:start) do |*args, **kwargs|
       original_start.call(*args, **kwargs)
     end
+    Crawlscope.reset!
   end
 
   def test_validate_passes_rake_arguments_to_cli
@@ -56,14 +59,43 @@ class CrawlscopeRakeTasksTest < Minitest::Test
     )
   end
 
+  def test_run_passes_the_shared_configuration_to_cli
+    calls = capture_cli_calls
+    configuration = Crawlscope.configuration
+
+    Crawlscope::RakeTasks.run("validate")
+
+    assert_same configuration, calls.fetch(0).fetch(:kwargs).fetch(:configuration)
+  end
+
+  def test_validate_passes_the_host_rule_registry_to_cli
+    registry = Crawlscope::RuleRegistry.new(rules: [Rule.new(:host)], default_codes: [:host])
+    Crawlscope.configure { |configuration| configuration.rule_registry = registry }
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.validate
+
+    call = calls.fetch(0)
+    assert_equal "validate", call.fetch(:argv).first
+    assert_same registry, call.fetch(:kwargs).fetch(:configuration).rule_registry
+  end
+
+  def test_run_exits_with_a_nonzero_cli_status
+    capture_cli_calls(status: 2)
+
+    error = assert_raises(SystemExit) { Crawlscope::RakeTasks.run("validate") }
+
+    assert_equal 2, error.status
+  end
+
   private
 
-  def capture_cli_calls
+  def capture_cli_calls(status: 0)
     calls = []
     singleton_class = class << Crawlscope::Cli; self; end
     singleton_class.define_method(:start) do |argv, **kwargs|
       calls << {argv: argv, kwargs: kwargs}
-      0
+      status
     end
     calls
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -259,6 +259,8 @@ files:
 - lib/crawlscope/tasks.rb
 - lib/crawlscope/url.rb
 - lib/crawlscope/version.rb
+- lib/generators/crawlscope/install_generator.rb
+- lib/generators/crawlscope/templates/initializer.rb.tt
 - lib/tasks/crawlscope_tasks.rake
 - test/crawlscope/browser_test.rb
 - test/crawlscope/cli_test.rb
@@ -269,6 +271,7 @@ files:
 - test/crawlscope/fetch_executor_test.rb
 - test/crawlscope/http_test.rb
 - test/crawlscope/indexability_rule_test.rb
+- test/crawlscope/install_generator_test.rb
 - test/crawlscope/links_rule_test.rb
 - test/crawlscope/loader_test.rb
 - test/crawlscope/metadata_rule_test.rb