crawlscope 0.3.0 → 0.5.0

data/lib/crawlscope/sitemap.rb

@@ -25,6 +25,9 @@ module Crawlscope
       visited.add(source)
       document = Nokogiri::XML(read(source))
       root_name = document.root&.name
+      unless %w[sitemapindex urlset].include?(root_name)
+        raise ValidationError, "Sitemap #{source} has unexpected root #{root_name.inspect}"
+      end
 
       if root_name == "sitemapindex"
         document.xpath("//xmlns:sitemap/xmlns:loc", SITEMAP_NAMESPACE).flat_map do |node|
@@ -40,7 +43,12 @@ module Crawlscope
 
     def read(source)
       if Url.remote?(source)
-        connection.get(source).body
+        response = connection.get(source)
+        unless response.status.to_i.between?(200, 299)
+          raise ValidationError, "Sitemap #{source} returned HTTP #{response.status}"
+        end
+
+        response.body
       else
         File.read(source)
       end

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/tasks/crawlscope_tasks.rake

@@ -1,33 +1,43 @@
 namespace :crawlscope do
-  desc "Validate URLs with all default Crawlscope rules. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
-  task validate: :environment do
-    Crawlscope::RakeTasks.validate
+  desc "Validate URLs with all default Crawlscope rules. Args: [url,sitemap,rules]. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
+  task :validate, [:url, :sitemap, :rules] => :environment do |_task, args|
+    Crawlscope::RakeTasks.validate(url: args[:url], sitemap_path: args[:sitemap], rule_names: args[:rules])
   end
 
   namespace :validate do
-    desc "Directly validate JSON-LD on one or more URLs. ENV: URL (semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
-    task ldjson: :environment do
-      Crawlscope::RakeTasks.ldjson
+    desc "Directly validate JSON-LD on one URL. Args: [url]. ENV: URL (semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
+    task :ldjson, [:url] => :environment do |_task, args|
+      Crawlscope::RakeTasks.ldjson(urls: args[:url])
     end
 
-    desc "Validate URLs with the metadata rule. ENV: URL, SITEMAP, JS=1"
-    task metadata: :environment do
-      Crawlscope::RakeTasks.validate_rule("metadata")
+    desc "Validate URLs with the indexability rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :indexability, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("indexability", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate sitemap URLs with the structured_data rule. ENV: URL, SITEMAP, JS=1"
-    task structured_data: :environment do
-      Crawlscope::RakeTasks.validate_rule("structured_data")
+    desc "Validate URLs with the metadata rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :metadata, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("metadata", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the uniqueness rule. ENV: URL, SITEMAP, JS=1"
-    task uniqueness: :environment do
-      Crawlscope::RakeTasks.validate_rule("uniqueness")
+    desc "Validate sitemap URLs with the structured_data rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :structured_data, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("structured_data", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the links rule. ENV: URL, SITEMAP, JS=1"
-    task links: :environment do
-      Crawlscope::RakeTasks.validate_rule("links")
+    desc "Validate URLs with the uniqueness rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :uniqueness, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("uniqueness", url: args[:url], sitemap_path: args[:sitemap])
+    end
+
+    desc "Validate URLs with the content_quality rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :content_quality, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("content_quality", url: args[:url], sitemap_path: args[:sitemap])
+    end
+
+    desc "Validate URLs with the links rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :links, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("links", url: args[:url], sitemap_path: args[:sitemap])
     end
   end
 end

data/test/crawlscope/cli_test.rb

@@ -265,6 +265,7 @@ class CrawlscopeCliTest < Minitest::Test
 
     assert_equal 1, status
     assert_includes err.string, "No URLs found in sitemap"
+    refute_includes err.string, "Usage:"
   end
 
   private

data/test/crawlscope/configuration_test.rb

@@ -20,7 +20,14 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "https://example.com", audit.instance_variable_get(:@base_url)
     assert_equal "/tmp/sitemap.xml", audit.instance_variable_get(:@sitemap_path)
     assert_equal 4, audit.instance_variable_get(:@concurrency)
-    assert_equal %i[metadata structured_data uniqueness links], audit.instance_variable_get(:@rules).map(&:code)
+    assert_equal %i[
+      indexability
+      metadata
+      structured_data
+      uniqueness
+      content_quality
+      links
+    ], audit.instance_variable_get(:@rules).map(&:code)
   end
 
   def test_audit_raises_without_base_url

data/test/crawlscope/content_quality_rule_test.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeContentQualityRuleTest < Minitest::Test
+  def test_reports_thin_visible_text_and_low_html_text_ratio
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(main: "Short page <div>#{"<span></span>" * 500}</div>")
+
+    Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :thin_visible_text
+    assert_includes codes, :low_visible_text_ratio
+  end
+
+  def test_visible_text_ratio_ignores_markup_outside_main_content
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      main: Array.new(260) { |index| "word#{index}" }.join(" "),
+      head_markup: "<style>#{"body{}" * 10_000}</style>",
+      extra_markup: "<nav>#{"<a href=\"/\">Navigation</a>" * 500}</nav>"
+    )
+
+    Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    refute_includes issues.to_a.map(&:code), :low_visible_text_ratio
+  end
+
+  def test_reports_low_unique_token_ratio_for_repetitive_content
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(main: ("hotel location service " * 100).strip)
+
+    Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    issue = issues.to_a.find { |item| item.code == :low_unique_token_ratio }
+    assert issue
+    assert_operator issue.details[:ratio], :<, issue.details[:threshold]
+  end
+
+  private
+
+  def page_with(main:, extra_markup: "", head_markup: "")
+    body = <<~HTML
+      <html>
+        <head>
+          <title>Content quality</title>
+          #{head_markup}
+        </head>
+        <body>
+          #{extra_markup}
+          <main>#{main}</main>
+        </body>
+      </html>
+    HTML
+
+    Crawlscope::Page.new(
+      url: "https://example.com/page",
+      normalized_url: "https://example.com/page",
+      final_url: "https://example.com/page",
+      normalized_final_url: "https://example.com/page",
+      status: 200,
+      headers: {"content-type" => "text/html"},
+      body: body,
+      doc: Nokogiri::HTML(body)
+    )
+  end
+end

data/test/crawlscope/crawl_test.rb

@@ -45,6 +45,7 @@ class CrawlscopeCrawlTest < Minitest::Test
             <body>
               <main>
                 <h1>Pricing</h1>
+                <p>#{Array.new(260) { |index| "pricing#{index}" }.join(" ")}</p>
               </main>
             </body>
           </html>
@@ -100,7 +101,15 @@ class CrawlscopeCrawlTest < Minitest::Test
     ).call
 
     refute result.ok?
-    assert_equal %i[incomplete_open_graph_tags meta_description_too_long missing_canonical missing_h1 missing_structured_data title_repeats_site_name].sort, result.issues.to_a.map(&:code).uniq.sort
+    assert_equal %i[
+      incomplete_open_graph_tags
+      meta_description_too_long
+      missing_canonical
+      missing_h1
+      missing_structured_data
+      thin_visible_text
+      title_repeats_site_name
+    ].sort, result.issues.to_a.map(&:code).uniq.sort
   end
 
   def test_uses_browser_when_renderer_is_browser
@@ -147,6 +156,7 @@ class CrawlscopeCrawlTest < Minitest::Test
             <body>
               <main>
                 <h1>Pricing</h1>
+                <p>#{Array.new(260) { |index| "pricing#{index}" }.join(" ")}</p>
               </main>
             </body>
           </html>
@@ -178,4 +188,30 @@ class CrawlscopeCrawlTest < Minitest::Test
     assert_equal ["https://example.com/pricing"], fake_browser.urls
     assert fake_browser.closed
   end
+
+  def test_reports_sitemap_redirect_url
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/old</loc></url>
+        </urlset>
+      XML
+    )
+
+    stub_request(:get, "https://example.com/old")
+      .to_return(status: 301, headers: {"Location" => "https://example.com/new"}, body: "")
+    stub_request(:get, "https://example.com/new")
+      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<html><body>Moved</body></html>")
+
+    result = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default
+    ).call
+
+    assert_includes result.issues.to_a.map(&:code), :sitemap_redirect_url
+  end
 end

data/test/crawlscope/indexability_rule_test.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeIndexabilityRuleTest < Minitest::Test
+  def test_reports_meta_noindex
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      body: <<~HTML
+        <html>
+          <head><meta name="robots" content="noindex, follow"></head>
+          <body><main>Visible content</main></body>
+        </html>
+      HTML
+    )
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    issue = issues.to_a.fetch(0)
+    assert_equal :noindex_meta, issue.code
+    assert_equal :error, issue.severity
+    assert_equal "noindex, follow", issue.details[:content]
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :noindex_follow_meta
+    assert_includes codes, :sitemap_noindex_url
+  end
+
+  def test_reports_meta_nofollow
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      body: <<~HTML
+        <html>
+          <head><meta name="robots" content="nofollow"></head>
+          <body><main>Visible content</main></body>
+        </html>
+      HTML
+    )
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    assert_equal [:nofollow_meta], issues.to_a.map(&:code)
+  end
+
+  def test_reports_noindex_nofollow_header
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(headers: {"X-Robots-Tag" => "googlebot: noindex, nofollow"})
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :noindex_header
+    assert_includes codes, :nofollow_header
+    assert_includes codes, :noindex_nofollow_header
+    assert_includes codes, :sitemap_noindex_url
+  end
+
+  def test_reports_x_robots_tag_noindex
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(headers: {"X-Robots-Tag" => "noindex"})
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    issue = issues.to_a.fetch(0)
+    assert_equal :noindex_header, issue.code
+    assert_equal :error, issue.severity
+    assert_equal "noindex", issue.details[:content]
+  end
+
+  def test_reports_x_robots_tag_noindex_for_non_html_response
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      body: "%PDF-1.7",
+      doc: nil,
+      headers: {"content-type" => "application/pdf", "X-Robots-Tag" => "noindex"}
+    )
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    issue = issues.to_a.fetch(0)
+    assert_equal :noindex_header, issue.code
+    assert_equal :error, issue.severity
+    assert_equal "noindex", issue.details[:content]
+  end
+
+  def test_reports_scoped_x_robots_tag_noindex
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(headers: {"X-Robots-Tag" => "googlebot: noindex, nofollow"})
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    issue = issues.to_a.fetch(0)
+    assert_equal :noindex_header, issue.code
+    assert_equal "googlebot: noindex, nofollow", issue.details[:content]
+  end
+
+  def test_reports_x_robots_tag_none
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(headers: {"X-Robots-Tag" => "none"})
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    issue = issues.to_a.fetch(0)
+    assert_equal :noindex_header, issue.code
+    assert_equal "none", issue.details[:content]
+  end
+
+  private
+
+  def page_with(body: nil, doc: :parse, headers: {"content-type" => "text/html"})
+    body ||= <<~HTML
+      <html>
+        <head><title>Indexable</title></head>
+        <body><main>Visible content</main></body>
+      </html>
+    HTML
+
+    Crawlscope::Page.new(
+      url: "https://example.com/page",
+      normalized_url: "https://example.com/page",
+      final_url: "https://example.com/page",
+      normalized_final_url: "https://example.com/page",
+      status: 200,
+      headers: headers,
+      body: body,
+      doc: (doc == :parse) ? Nokogiri::HTML(body) : doc
+    )
+  end
+end

data/test/crawlscope/links_rule_test.rb

@@ -41,7 +41,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
       context: context
     )
 
-    assert_equal [:broken_internal_link], issues.to_a.map(&:code)
+    assert_includes issues.to_a.map(&:code), :broken_internal_link
     assert_includes issues.to_a.first.message, "HTTP 404"
   end
 
@@ -114,8 +114,151 @@ class CrawlscopeLinksRuleTest < Minitest::Test
       context: context
     )
 
-    assert_equal [:low_inbound_anchor_links], issues.to_a.map(&:code)
-    assert_equal "https://example.com/guide", issues.to_a.first.url
+    orphan_issue = issues.to_a.find { |item| item.code == :orphan_page }
+    assert orphan_issue
+    assert_includes issues.to_a.map(&:code), :low_dofollow_inlinks
+    assert_equal "https://example.com/guide", orphan_issue.url
+  end
+
+  def test_reports_pages_with_no_outgoing_internal_links
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context
+    )
+
+    issue = issues.to_a.find { |item| item.code == :page_has_no_outgoing_links }
+    assert issue
+    assert_equal "https://example.com/pricing", issue.url
+  end
+
+  def test_reports_nofollow_outlinks_and_inlink_follow_mix
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing", "https://example.com/about"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\" rel=\"nofollow\">Pricing</a><a href=\"/about\">About</a></main>"),
+        page(url: "https://example.com/about", body: "<main><a href=\"/pricing\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :nofollow_internal_outlinks
+    assert_includes codes, :mixed_follow_internal_inlinks
+  end
+
+  def test_reports_only_nofollow_internal_inlinks
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\" rel=\"nofollow\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+
+    assert_includes issues.to_a.map(&:code), :only_nofollow_internal_inlinks
+  end
+
+  def test_reports_https_pages_linking_to_internal_http_urls
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"http://example.com/pricing\">Pricing</a></main>")],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+
+    assert_includes issues.to_a.map(&:code), :http_internal_link
+  end
+
+  def test_reports_canonical_target_link_issues
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      redirects = target_url == "https://example.com/canonical-about"
+      status = redirects ? 301 : 200
+      final_url = redirects ? "https://example.com/about" : target_url
+      {crawled: false, error: nil, final_url: final_url, status: status}
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/about"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/about\">About</a></main>"),
+        page(
+          url: "https://example.com/about",
+          body: <<~HTML
+            <html>
+              <head><link rel="canonical" href="https://example.com/canonical-about"></head>
+              <body><main><p>About</p></main></body>
+            </html>
+          HTML
+        )
+      ],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :canonical_no_internal_inlinks
+    assert_includes codes, :canonical_points_to_redirect
+  end
+
+  def test_reports_indexable_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        error: nil,
+        final_url: target_url,
+        html: true,
+        status: 200
+      }
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    issue = issues.to_a.find { |item| item.code == :indexable_page_missing_from_sitemap }
+    assert issue
+    assert_equal "https://example.com/hidden", issue.url
+  end
+
+  def test_reports_url_hygiene_issues
+    issues = Crawlscope::IssueCollection.new
+    long_path = "a" * 2_050
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com//bad", "https://example.com/#{long_path}"],
+      pages: [
+        page(url: "https://example.com//bad", body: "<main><a href=\"/ok\">OK</a></main>"),
+        page(url: "https://example.com/#{long_path}", body: "<main><a href=\"/ok\">OK</a></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: false, error: nil, final_url: target_url, html: true, status: 200} })
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :url_double_slash
+    assert_includes codes, :url_too_long
   end
 
   def test_counts_root_page_links_as_inbound_links
@@ -217,6 +360,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
         crawled: true,
         error: nil,
         final_url: target_url,
+        html: true,
         status: 200
       }
     when "https://example.com/missing"
@@ -224,6 +368,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
         crawled: false,
         error: nil,
         final_url: target_url,
+        html: false,
         status: 404
       }
     end

data/test/crawlscope/metadata_rule_test.rb

@@ -48,6 +48,42 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     refute_includes issues.to_a.map(&:code), :canonical_mismatch
   end
 
+  def test_reports_multiple_title_multiple_descriptions_empty_h1_and_sitemap_canonical_mismatch
+    issues = Crawlscope::IssueCollection.new
+    invalid_page = page(
+      body: <<~HTML
+        <html>
+          <head>
+            <title>About</title>
+            <title>Duplicate About</title>
+            <meta name="description" content="A clear description that is long enough for search snippets, local validation checks, and realistic production metadata audits.">
+            <meta name="description" content="Duplicate description">
+            <link rel="canonical" href="https://example.com/canonical-about">
+            <meta property="og:title" content="About">
+            <meta property="og:description" content="About page">
+            <meta property="og:url" content="https://example.com/about">
+            <meta property="og:type" content="website">
+            <meta property="og:image" content="https://example.com/icon.png">
+          </head>
+          <body><main><h1> </h1></main></body>
+        </html>
+      HTML
+    )
+
+    Crawlscope::Rules::Metadata.new.call(
+      urls: [invalid_page.url],
+      pages: [invalid_page],
+      issues: issues
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :multiple_title_tags
+    assert_includes codes, :multiple_meta_descriptions
+    assert_includes codes, :empty_h1
+    assert_includes codes, :canonical_mismatch
+    assert_includes codes, :non_canonical_page_in_sitemap
+  end
+
   private
 
   def page(url: "https://example.com/about", body: nil)

data/test/crawlscope/rake_tasks_test.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeRakeTasksTest < Minitest::Test
+  def setup
+    @original_start = Crawlscope::Cli.method(:start)
+  end
+
+  def teardown
+    singleton_class = class << Crawlscope::Cli; self; end
+    original_start = @original_start
+    singleton_class.define_method(:start) do |*args, **kwargs|
+      original_start.call(*args, **kwargs)
+    end
+  end
+
+  def test_validate_passes_rake_arguments_to_cli
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.validate(
+      url: "http://localhost:3001",
+      sitemap_path: "http://localhost:3001/sitemap.xml",
+      rule_names: "metadata,links"
+    )
+
+    assert_equal(
+      ["validate", "--url", "http://localhost:3001", "--sitemap", "http://localhost:3001/sitemap.xml", "--rules", "metadata,links"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+
+  def test_validate_rule_passes_rule_and_rake_arguments_to_cli
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.validate_rule(
+      "metadata",
+      url: "http://localhost:3001",
+      sitemap_path: "http://localhost:3001/sitemap.xml"
+    )
+
+    assert_equal(
+      ["validate", "--url", "http://localhost:3001", "--sitemap", "http://localhost:3001/sitemap.xml", "--rules", "metadata"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+
+  def test_ldjson_passes_rake_url_argument_to_cli
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.ldjson(urls: "http://localhost:3001/article")
+
+    assert_equal(
+      ["ldjson", "--url", "http://localhost:3001/article"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+
+  private
+
+  def capture_cli_calls
+    calls = []
+    singleton_class = class << Crawlscope::Cli; self; end
+    singleton_class.define_method(:start) do |argv, **kwargs|
+      calls << {argv: argv, kwargs: kwargs}
+      0
+    end
+    calls
+  end
+end