crawlscope 0.4.0 → 0.5.0

data/lib/tasks/crawlscope_tasks.rake

@@ -1,43 +1,43 @@
 namespace :crawlscope do
-  desc "Validate URLs with all default Crawlscope rules. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
-  task validate: :environment do
-    Crawlscope::RakeTasks.validate
+  desc "Validate URLs with all default Crawlscope rules. Args: [url,sitemap,rules]. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
+  task :validate, [:url, :sitemap, :rules] => :environment do |_task, args|
+    Crawlscope::RakeTasks.validate(url: args[:url], sitemap_path: args[:sitemap], rule_names: args[:rules])
   end
 
   namespace :validate do
-    desc "Directly validate JSON-LD on one or more URLs. ENV: URL (semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
-    task ldjson: :environment do
-      Crawlscope::RakeTasks.ldjson
+    desc "Directly validate JSON-LD on one URL. Args: [url]. ENV: URL (semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
+    task :ldjson, [:url] => :environment do |_task, args|
+      Crawlscope::RakeTasks.ldjson(urls: args[:url])
     end
 
-    desc "Validate URLs with the indexability rule. ENV: URL, SITEMAP, JS=1"
-    task indexability: :environment do
-      Crawlscope::RakeTasks.validate_rule("indexability")
+    desc "Validate URLs with the indexability rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :indexability, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("indexability", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the metadata rule. ENV: URL, SITEMAP, JS=1"
-    task metadata: :environment do
-      Crawlscope::RakeTasks.validate_rule("metadata")
+    desc "Validate URLs with the metadata rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :metadata, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("metadata", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate sitemap URLs with the structured_data rule. ENV: URL, SITEMAP, JS=1"
-    task structured_data: :environment do
-      Crawlscope::RakeTasks.validate_rule("structured_data")
+    desc "Validate sitemap URLs with the structured_data rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :structured_data, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("structured_data", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the uniqueness rule. ENV: URL, SITEMAP, JS=1"
-    task uniqueness: :environment do
-      Crawlscope::RakeTasks.validate_rule("uniqueness")
+    desc "Validate URLs with the uniqueness rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :uniqueness, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("uniqueness", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the content_quality rule. ENV: URL, SITEMAP, JS=1"
-    task content_quality: :environment do
-      Crawlscope::RakeTasks.validate_rule("content_quality")
+    desc "Validate URLs with the content_quality rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :content_quality, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("content_quality", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the links rule. ENV: URL, SITEMAP, JS=1"
-    task links: :environment do
-      Crawlscope::RakeTasks.validate_rule("links")
+    desc "Validate URLs with the links rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :links, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("links", url: args[:url], sitemap_path: args[:sitemap])
     end
   end
 end

data/test/crawlscope/cli_test.rb

@@ -265,6 +265,7 @@ class CrawlscopeCliTest < Minitest::Test
 
     assert_equal 1, status
     assert_includes err.string, "No URLs found in sitemap"
+    refute_includes err.string, "Usage:"
   end
 
   private

data/test/crawlscope/crawl_test.rb

@@ -188,4 +188,30 @@ class CrawlscopeCrawlTest < Minitest::Test
     assert_equal ["https://example.com/pricing"], fake_browser.urls
     assert fake_browser.closed
   end
+
+  def test_reports_sitemap_redirect_url
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/old</loc></url>
+        </urlset>
+      XML
+    )
+
+    stub_request(:get, "https://example.com/old")
+      .to_return(status: 301, headers: {"Location" => "https://example.com/new"}, body: "")
+    stub_request(:get, "https://example.com/new")
+      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<html><body>Moved</body></html>")
+
+    result = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default
+    ).call
+
+    assert_includes result.issues.to_a.map(&:code), :sitemap_redirect_url
+  end
 end

data/test/crawlscope/indexability_rule_test.rb

@@ -20,6 +20,39 @@ class CrawlscopeIndexabilityRuleTest < Minitest::Test
     assert_equal :noindex_meta, issue.code
     assert_equal :error, issue.severity
     assert_equal "noindex, follow", issue.details[:content]
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :noindex_follow_meta
+    assert_includes codes, :sitemap_noindex_url
+  end
+
+  def test_reports_meta_nofollow
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      body: <<~HTML
+        <html>
+          <head><meta name="robots" content="nofollow"></head>
+          <body><main>Visible content</main></body>
+        </html>
+      HTML
+    )
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    assert_equal [:nofollow_meta], issues.to_a.map(&:code)
+  end
+
+  def test_reports_noindex_nofollow_header
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(headers: {"X-Robots-Tag" => "googlebot: noindex, nofollow"})
+
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :noindex_header
+    assert_includes codes, :nofollow_header
+    assert_includes codes, :noindex_nofollow_header
+    assert_includes codes, :sitemap_noindex_url
   end
 
   def test_reports_x_robots_tag_noindex

data/test/crawlscope/links_rule_test.rb

@@ -41,7 +41,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
       context: context
     )
 
-    assert_equal [:broken_internal_link], issues.to_a.map(&:code)
+    assert_includes issues.to_a.map(&:code), :broken_internal_link
     assert_includes issues.to_a.first.message, "HTTP 404"
   end
 
@@ -114,8 +114,151 @@ class CrawlscopeLinksRuleTest < Minitest::Test
       context: context
     )
 
-    assert_equal [:low_inbound_anchor_links], issues.to_a.map(&:code)
-    assert_equal "https://example.com/guide", issues.to_a.first.url
+    orphan_issue = issues.to_a.find { |item| item.code == :orphan_page }
+    assert orphan_issue
+    assert_includes issues.to_a.map(&:code), :low_dofollow_inlinks
+    assert_equal "https://example.com/guide", orphan_issue.url
+  end
+
+  def test_reports_pages_with_no_outgoing_internal_links
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context
+    )
+
+    issue = issues.to_a.find { |item| item.code == :page_has_no_outgoing_links }
+    assert issue
+    assert_equal "https://example.com/pricing", issue.url
+  end
+
+  def test_reports_nofollow_outlinks_and_inlink_follow_mix
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing", "https://example.com/about"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\" rel=\"nofollow\">Pricing</a><a href=\"/about\">About</a></main>"),
+        page(url: "https://example.com/about", body: "<main><a href=\"/pricing\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :nofollow_internal_outlinks
+    assert_includes codes, :mixed_follow_internal_inlinks
+  end
+
+  def test_reports_only_nofollow_internal_inlinks
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\" rel=\"nofollow\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+
+    assert_includes issues.to_a.map(&:code), :only_nofollow_internal_inlinks
+  end
+
+  def test_reports_https_pages_linking_to_internal_http_urls
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"http://example.com/pricing\">Pricing</a></main>")],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+
+    assert_includes issues.to_a.map(&:code), :http_internal_link
+  end
+
+  def test_reports_canonical_target_link_issues
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      redirects = target_url == "https://example.com/canonical-about"
+      status = redirects ? 301 : 200
+      final_url = redirects ? "https://example.com/about" : target_url
+      {crawled: false, error: nil, final_url: final_url, status: status}
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/about"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/about\">About</a></main>"),
+        page(
+          url: "https://example.com/about",
+          body: <<~HTML
+            <html>
+              <head><link rel="canonical" href="https://example.com/canonical-about"></head>
+              <body><main><p>About</p></main></body>
+            </html>
+          HTML
+        )
+      ],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :canonical_no_internal_inlinks
+    assert_includes codes, :canonical_points_to_redirect
+  end
+
+  def test_reports_indexable_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        error: nil,
+        final_url: target_url,
+        html: true,
+        status: 200
+      }
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    issue = issues.to_a.find { |item| item.code == :indexable_page_missing_from_sitemap }
+    assert issue
+    assert_equal "https://example.com/hidden", issue.url
+  end
+
+  def test_reports_url_hygiene_issues
+    issues = Crawlscope::IssueCollection.new
+    long_path = "a" * 2_050
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com//bad", "https://example.com/#{long_path}"],
+      pages: [
+        page(url: "https://example.com//bad", body: "<main><a href=\"/ok\">OK</a></main>"),
+        page(url: "https://example.com/#{long_path}", body: "<main><a href=\"/ok\">OK</a></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: false, error: nil, final_url: target_url, html: true, status: 200} })
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :url_double_slash
+    assert_includes codes, :url_too_long
   end
 
   def test_counts_root_page_links_as_inbound_links
@@ -217,6 +360,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
         crawled: true,
         error: nil,
         final_url: target_url,
+        html: true,
         status: 200
       }
     when "https://example.com/missing"
@@ -224,6 +368,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
         crawled: false,
         error: nil,
         final_url: target_url,
+        html: false,
         status: 404
       }
     end

data/test/crawlscope/metadata_rule_test.rb

@@ -48,6 +48,42 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     refute_includes issues.to_a.map(&:code), :canonical_mismatch
   end
 
+  def test_reports_multiple_title_multiple_descriptions_empty_h1_and_sitemap_canonical_mismatch
+    issues = Crawlscope::IssueCollection.new
+    invalid_page = page(
+      body: <<~HTML
+        <html>
+          <head>
+            <title>About</title>
+            <title>Duplicate About</title>
+            <meta name="description" content="A clear description that is long enough for search snippets, local validation checks, and realistic production metadata audits.">
+            <meta name="description" content="Duplicate description">
+            <link rel="canonical" href="https://example.com/canonical-about">
+            <meta property="og:title" content="About">
+            <meta property="og:description" content="About page">
+            <meta property="og:url" content="https://example.com/about">
+            <meta property="og:type" content="website">
+            <meta property="og:image" content="https://example.com/icon.png">
+          </head>
+          <body><main><h1> </h1></main></body>
+        </html>
+      HTML
+    )
+
+    Crawlscope::Rules::Metadata.new.call(
+      urls: [invalid_page.url],
+      pages: [invalid_page],
+      issues: issues
+    )
+
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :multiple_title_tags
+    assert_includes codes, :multiple_meta_descriptions
+    assert_includes codes, :empty_h1
+    assert_includes codes, :canonical_mismatch
+    assert_includes codes, :non_canonical_page_in_sitemap
+  end
+
   private
 
   def page(url: "https://example.com/about", body: nil)

data/test/crawlscope/rake_tasks_test.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeRakeTasksTest < Minitest::Test
+  def setup
+    @original_start = Crawlscope::Cli.method(:start)
+  end
+
+  def teardown
+    singleton_class = class << Crawlscope::Cli; self; end
+    original_start = @original_start
+    singleton_class.define_method(:start) do |*args, **kwargs|
+      original_start.call(*args, **kwargs)
+    end
+  end
+
+  def test_validate_passes_rake_arguments_to_cli
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.validate(
+      url: "http://localhost:3001",
+      sitemap_path: "http://localhost:3001/sitemap.xml",
+      rule_names: "metadata,links"
+    )
+
+    assert_equal(
+      ["validate", "--url", "http://localhost:3001", "--sitemap", "http://localhost:3001/sitemap.xml", "--rules", "metadata,links"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+
+  def test_validate_rule_passes_rule_and_rake_arguments_to_cli
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.validate_rule(
+      "metadata",
+      url: "http://localhost:3001",
+      sitemap_path: "http://localhost:3001/sitemap.xml"
+    )
+
+    assert_equal(
+      ["validate", "--url", "http://localhost:3001", "--sitemap", "http://localhost:3001/sitemap.xml", "--rules", "metadata"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+
+  def test_ldjson_passes_rake_url_argument_to_cli
+    calls = capture_cli_calls
+
+    Crawlscope::RakeTasks.ldjson(urls: "http://localhost:3001/article")
+
+    assert_equal(
+      ["ldjson", "--url", "http://localhost:3001/article"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+
+  private
+
+  def capture_cli_calls
+    calls = []
+    singleton_class = class << Crawlscope::Cli; self; end
+    singleton_class.define_method(:start) do |argv, **kwargs|
+      calls << {argv: argv, kwargs: kwargs}
+      0
+    end
+    calls
+  end
+end

data/test/crawlscope/reporter_test.rb

@@ -23,7 +23,7 @@ class CrawlscopeReporterTest < Minitest::Test
     refute_includes output, "Status: FAILED"
   end
 
-  def test_reports_failed_result_with_severity_counts
+  def test_reports_failed_result_with_grouped_counts_and_offenses
     io = StringIO.new
     issues = Crawlscope::IssueCollection.new
     issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com/a", message: "missing <title>", details: {})
@@ -42,9 +42,13 @@ class CrawlscopeReporterTest < Minitest::Test
 
     assert_includes output, "Status: FAILED"
     assert_includes output, "Issues: 2"
+    assert_includes output, "Severity:"
     assert_includes output, "notice: 1"
     assert_includes output, "warning: 1"
-    assert_includes output, "- [warning] https://example.com/a missing <title>"
-    assert_includes output, "- [notice] https://example.com/b broken internal link"
+    assert_includes output, "Category:"
+    assert_includes output, "links: 1"
+    assert_includes output, "metadata: 1"
+    assert_includes output, "  - [warning] missing_title https://example.com/a missing <title>"
+    assert_includes output, "  - [notice] broken_internal_link https://example.com/b broken internal link"
   end
 end

data/test/crawlscope/sitemap_test.rb

@@ -49,6 +49,30 @@ class CrawlscopeSitemapTest < Minitest::Test
     assert_equal ["https://www.example.com/features/reviews"], parser.urls(base_url: "https://www.example.com")
   end
 
+  def test_remote_sitemap_http_error_is_explicit
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(status: 500, body: "<html><body>Error</body></html>")
+
+    parser = Crawlscope::Sitemap.new(path: "https://www.example.com/sitemap.xml")
+
+    error = assert_raises(Crawlscope::ValidationError) do
+      parser.urls(base_url: "https://www.example.com")
+    end
+    assert_equal "Sitemap https://www.example.com/sitemap.xml returned HTTP 500", error.message
+  end
+
+  def test_invalid_sitemap_root_is_explicit
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(status: 200, body: "<html><body>Error</body></html>")
+
+    parser = Crawlscope::Sitemap.new(path: "https://www.example.com/sitemap.xml")
+
+    error = assert_raises(Crawlscope::ValidationError) do
+      parser.urls(base_url: "https://www.example.com")
+    end
+    assert_equal 'Sitemap https://www.example.com/sitemap.xml has unexpected root "html"', error.message
+  end
+
   def test_rebases_remote_sitemap_index_children_to_base_url
     stub_request(:get, "http://localhost:3000/sitemap.xml")
       .to_return(

data/test/crawlscope/structured_data_rule_test.rb

@@ -79,6 +79,62 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
     assert_equal ["json-ld", "microdata"], issues.to_a.first.details[:expected_sources]
   end
 
+  def test_reports_structured_data_missing_type
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","headline":"Untyped article"}
+            </script>
+          </head>
+          <body><h1>Article</h1></body>
+        </html>
+      HTML
+    )
+
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+
+    assert_includes issues.to_a.map(&:code), :structured_data_missing_type
+  end
+
+  def test_reports_graph_entries_missing_type
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","@type":"WebPage","@graph":[{"name":"Untyped node"}]}
+            </script>
+          </head>
+          <body><h1>Article</h1></body>
+        </html>
+      HTML
+    )
+
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+
+    issue = issues.to_a.find { |item| item.code == :structured_data_missing_type }
+    assert issue
+    assert_equal ["$.@graph[0]"], issue.details[:paths]
+  end
+
   def test_validates_job_posting_markup
     issues = Crawlscope::IssueCollection.new
     rule = Crawlscope::Rules::StructuredData.new

data/test/crawlscope/uniqueness_rule_test.rb

@@ -13,7 +13,20 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
 
     rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
 
-    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_title].sort, issues.to_a.map(&:code).sort
+    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_pages_without_canonical duplicate_title].sort, issues.to_a.map(&:code).sort
+  end
+
+  def test_allows_duplicate_pages_when_canonicals_are_present
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::Uniqueness.new
+    pages = [
+      page(url: "https://example.com/a", canonical: "https://example.com/a"),
+      page(url: "https://example.com/b", canonical: "https://example.com/a")
+    ]
+
+    rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+
+    refute_includes issues.to_a.map(&:code), :duplicate_pages_without_canonical
   end
 
   def test_reports_near_duplicate_content
@@ -59,13 +72,15 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     TEXT
   end
 
-  def page(url:, content: nil)
+  def page(url:, content: nil, canonical: nil)
     repeated_text = content || ("Useful content " * 30).strip
+    canonical_tag = canonical ? %(<link rel="canonical" href="#{canonical}">) : ""
     body = <<~HTML
       <html>
         <head>
           <title>Example Title</title>
           <meta name="description" content="Example description">
+          #{canonical_tag}
         </head>
         <body>
           <main>#{repeated_text}</main>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -239,6 +239,7 @@ files:
 - test/crawlscope/links_rule_test.rb
 - test/crawlscope/loader_test.rb
 - test/crawlscope/metadata_rule_test.rb
+- test/crawlscope/rake_tasks_test.rb
 - test/crawlscope/reporter_test.rb
 - test/crawlscope/rule_registry_test.rb
 - test/crawlscope/run_test.rb