crawlscope 0.7.2 → 0.8.1

data/test/crawlscope/configuration_test.rb

@@ -14,6 +14,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
       config.site_name = -> { "Example" }
       config.concurrency = -> { 4 }
       config.fetch_executor = -> { :threaded }
+      config.profile_token = -> { "profile-token" }
     end
 
     audit = Crawlscope.configuration.audit
@@ -22,6 +23,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "/tmp/sitemap.xml", audit.instance_variable_get(:@sitemap_path)
     assert_equal 4, audit.instance_variable_get(:@concurrency)
     assert_equal :threaded, audit.instance_variable_get(:@fetch_executor)
+    assert_equal "profile-token", audit.instance_variable_get(:@profile_token)
     assert_equal %i[
       indexability
       metadata
@@ -53,17 +55,35 @@ class CrawlscopeConfigurationTest < Minitest::Test
   end
 
   def test_defaults_are_normalized
-    config = Crawlscope::Configuration.new
+    with_profile_token(nil) do
+      config = Crawlscope::Configuration.new
+
+      assert_equal [200, 301, 302], config.allowed_statuses
+      assert_equal 10, config.concurrency
+      assert_equal :async, config.fetch_executor
+      assert_equal 4, config.browser_concurrency
+      assert_equal 5, config.network_idle_timeout_seconds
+      assert_equal :http, config.renderer
+      assert_equal 20, config.timeout_seconds
+      assert_equal $stdout, config.output
+      assert_nil config.profile_token
+      assert config.scroll_page?
+    end
+  end
 
-    assert_equal [200, 301, 302], config.allowed_statuses
-    assert_equal 10, config.concurrency
-    assert_equal :async, config.fetch_executor
-    assert_equal 4, config.browser_concurrency
-    assert_equal 5, config.network_idle_timeout_seconds
-    assert_equal :http, config.renderer
-    assert_equal 20, config.timeout_seconds
-    assert_equal $stdout, config.output
-    assert config.scroll_page?
+  def test_profile_token_defaults_to_the_environment
+    with_profile_token("environment-profile-token") do
+      assert_equal "environment-profile-token", Crawlscope::Configuration.new.profile_token
+    end
+  end
+
+  def test_configured_profile_token_precedes_the_environment
+    with_profile_token("environment-profile-token") do
+      config = Crawlscope::Configuration.new
+      config.profile_token = "configured-profile-token"
+
+      assert_equal "configured-profile-token", config.profile_token
+    end
   end
 
   def test_browser_renderer_defaults_to_threaded_fetch_executor
@@ -119,4 +139,24 @@ class CrawlscopeConfigurationTest < Minitest::Test
 
     assert_equal "Crawlscope concurrency must be an integer >= 1", error.message
   end
+
+  private
+
+  def with_profile_token(value)
+    previous_value = ENV["CRAWLSCOPE_PROFILE_TOKEN"]
+
+    if value.nil?
+      ENV.delete("CRAWLSCOPE_PROFILE_TOKEN")
+    else
+      ENV["CRAWLSCOPE_PROFILE_TOKEN"] = value
+    end
+
+    yield
+  ensure
+    if previous_value.nil?
+      ENV.delete("CRAWLSCOPE_PROFILE_TOKEN")
+    else
+      ENV["CRAWLSCOPE_PROFILE_TOKEN"] = previous_value
+    end
+  end
 end

data/test/crawlscope/crawl_test.rb

@@ -167,6 +167,35 @@ class CrawlscopeCrawlTest < Minitest::Test
     ].sort, result.issues.to_a.map(&:code).uniq.sort
   end
 
+  def test_profile_token_reaches_remote_sitemap_and_page_requests
+    sitemap_request = stub_request(:get, "https://example.com/sitemap.xml")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://example.com/about</loc></url>
+          </urlset>
+        XML
+      )
+    page_request = stub_request(:get, "https://example.com/about")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 200, body: "<html><body>About</body></html>")
+
+    Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: "https://example.com/sitemap.xml",
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded,
+      profile_token: "profile-token"
+    ).call
+
+    assert_requested sitemap_request
+    assert_requested page_request
+  end
+
   def test_uses_browser_when_renderer_is_browser
     File.write(
       @sitemap_path,
@@ -296,6 +325,42 @@ class CrawlscopeCrawlTest < Minitest::Test
     assert_includes result.issues.to_a.map(&:code), :sitemap_redirect_url
   end
 
+  def test_reports_redirects_without_duplicate_destination_content
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/old</loc></url>
+          <url><loc>https://example.com/alias</loc></url>
+          <url><loc>https://example.com/new</loc></url>
+        </urlset>
+      XML
+    )
+    ["old", "alias"].each do |path|
+      stub_request(:get, "https://example.com/#{path}")
+        .to_return(status: 301, headers: {"Location" => "https://example.com/new"}, body: "")
+    end
+    body = %(<html><head><title>Current page</title><meta name="description" content="Current description"><link rel="canonical" href="https://example.com/new"></head><body>#{"Useful content " * 30}</body></html>)
+    stub_request(:get, "https://example.com/new")
+      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: body)
+
+    result = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [Crawlscope::Rules::Uniqueness.new, Crawlscope::Rules::Metadata.new],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
+    ).call
+
+    assert_equal 3, result.pages.size
+    assert_equal ["https://example.com/old", "https://example.com/alias"], result.issues.select { |issue| issue.code == :sitemap_redirect_url }.map(&:url)
+    refute_includes result.issues.to_a.map(&:code), :redirected_page
+    assert_empty result.issues.select { |issue| issue.category == :uniqueness }
+    assert_empty result.issues.select { |issue| [:canonical_mismatch, :non_canonical_page_in_sitemap].include?(issue.code) }
+    assert_equal 1, result.issues.count { |issue| issue.code == :meta_description_too_short }
+  end
+
   def test_resolves_uncrawled_link_targets_as_a_bounded_batch
     File.write(
       @sitemap_path,

data/test/crawlscope/http_test.rb

@@ -5,13 +5,24 @@ require "test_helper"
 class CrawlscopeHttpTest < Minitest::Test
   def test_fetch_parses_html_response
     stub_request(:get, "https://example.com/page")
-      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<html><body>Hello</body></html>")
+      .to_return(
+        status: 200,
+        headers: {
+          "Content-Type" => "text/html",
+          "Server-Timing" => 'db;dur=12.5;desc="Primary database"'
+        },
+        body: "<html><body>Hello</body></html>"
+      )
 
     page = Crawlscope::Http.new(base_url: "https://example.com", timeout_seconds: 2).fetch("https://example.com/page")
 
     assert_equal 200, page.status
     assert page.html?
     assert_equal "Hello", page.doc.at_css("body").text
+    timing = page.server_timing
+    assert_same timing, page.server_timing
+    assert_equal 12.5, timing.first.duration
+    assert_equal "Primary database", timing.first.description
   end
 
   def test_fetch_parses_responses_without_content_type_as_html
@@ -23,6 +34,54 @@ class CrawlscopeHttpTest < Minitest::Test
     assert page.html?
   end
 
+  def test_fetch_sends_profile_token_to_same_origin_requests
+    request = stub_request(:get, "https://example.com/page")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 200, body: "<html></html>")
+
+    Crawlscope::Http.new(
+      base_url: "https://example.com",
+      timeout_seconds: 2,
+      profile_token: "profile-token"
+    ).fetch("https://example.com/page")
+
+    assert_requested request
+  end
+
+  def test_fetch_preserves_profile_token_on_same_origin_redirects
+    stub_request(:get, "https://example.com/start")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 302, headers: {"Location" => "/final"})
+    final_request = stub_request(:get, "https://example.com/final")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 200, body: "<html></html>")
+
+    Crawlscope::Http.new(
+      base_url: "https://example.com",
+      timeout_seconds: 2,
+      profile_token: "profile-token"
+    ).fetch("https://example.com/start")
+
+    assert_requested final_request
+  end
+
+  def test_fetch_removes_profile_token_from_cross_origin_redirects
+    stub_request(:get, "https://example.com/start")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 302, headers: {"Location" => "https://other.example/final"})
+    final_request = stub_request(:get, "https://other.example/final")
+      .with { |request| !request.headers.key?("X-Profile-Token") }
+      .to_return(status: 200, body: "<html></html>")
+
+    Crawlscope::Http.new(
+      base_url: "https://example.com",
+      timeout_seconds: 2,
+      profile_token: "profile-token"
+    ).fetch("https://example.com/start")
+
+    assert_requested final_request
+  end
+
   def test_fetch_leaves_non_html_response_unparsed
     stub_request(:get, "https://example.com/feed.xml")
       .to_return(status: 200, headers: {"content-type" => "application/xml"}, body: "<feed></feed>")

data/test/crawlscope/install_generator_test.rb

@@ -16,11 +16,14 @@ class CrawlscopeInstallGeneratorTest < Minitest::Test
       Crawlscope::Generators::InstallGenerator.start([], destination_root: destination)
 
       initializer = File.join(destination, "config/initializers/crawlscope.rb")
+      initializer_contents = File.read(initializer)
       rakefile = File.read(File.join(destination, "Rakefile"))
 
       assert File.exist?(initializer)
-      assert_includes File.read(initializer), "module CrawlscopeConfiguration"
-      assert_includes File.read(initializer), "CrawlscopeConfiguration.apply if defined?(Crawlscope)"
+      assert_includes initializer_contents, "module CrawlscopeConfiguration"
+      assert_includes initializer_contents, "CrawlscopeConfiguration.apply if defined?(Crawlscope)"
+      assert_includes initializer_contents, "\#{config.base_url.to_s.chomp(\"/\")}/sitemap.xml"
+      refute_includes initializer_contents, "Rails.public_path"
       assert_includes rakefile, 'require "crawlscope/tasks"'
       assert_operator rakefile.index('require "crawlscope/tasks"'), :<, rakefile.index('require_relative "config/application"')
 
@@ -28,14 +31,6 @@ class CrawlscopeInstallGeneratorTest < Minitest::Test
       assert status.success?, stderr
 
       script = <<~RUBY
-        require "pathname"
-
-        module Rails
-          def self.public_path
-            Pathname.new("public")
-          end
-        end
-
         require "crawlscope"
         load ARGV.fetch(0)
 
@@ -43,7 +38,10 @@ class CrawlscopeInstallGeneratorTest < Minitest::Test
         CrawlscopeConfiguration.apply
         abort "configuration replaced" unless Crawlscope.configuration.equal?(configuration)
         abort "base URL missing" unless configuration.base_url == "http://localhost:3000"
-        abort "sitemap missing" unless configuration.sitemap_path == "public/sitemap.xml"
+        abort "sitemap missing" unless configuration.sitemap_path == "http://localhost:3000/sitemap.xml"
+
+        ENV["CRAWLSCOPE_BASE_URL"] = "https://example.com"
+        abort "configured sitemap missing" unless configuration.sitemap_path == "https://example.com/sitemap.xml"
       RUBY
       _stdout, stderr, status = Open3.capture3(
         RbConfig.ruby,

data/test/crawlscope/links_rule_test.rb

@@ -56,7 +56,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     )
 
     assert_includes issues.to_a.map(&:code), :unresolved_internal_link
-    assert_includes issues.to_a.find { |issue| issue.code == :unresolved_internal_link }.message, "unable to validate internal link"
+    assert_includes issues.to_a.find { |issue| issue.code == :unresolved_internal_link }.message, "cannot check linked URL"
   end
 
   def test_ignores_fetch_errors_for_urls_already_crawled
@@ -429,6 +429,42 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     assert_empty issues.to_a
   end
 
+  def test_retains_all_link_sources_and_targets_in_issue_details
+    source_urls = 5.times.map { |index| "https://example.com/source-#{index}" }
+    target_urls = 5.times.map { |index| "https://example.com/target-#{index}" }
+    pages = source_urls.map do |url|
+      targets = target_urls.map { |target| %(<a href="#{target}" rel="nofollow">Target</a>) }.join
+      page(url: url, body: %(<html><body>#{targets}<a href="/missing">Missing</a><a href="/pricing">Pricing</a></body></html>))
+    end
+    resolver = lambda do |url|
+      {crawled: false, error: nil, final_url: url, html: true, status: url.end_with?("/missing") ? 404 : 200}
+    end
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(urls: source_urls, pages: pages, issues: issues, context: context(resolver: resolver))
+
+    broken = issues.find { |issue| issue.code == :broken_internal_link }
+    assert_equal source_urls, broken.details[:source_urls]
+    outgoing = issues.select { |issue| issue.code == :nofollow_internal_outlinks }
+    assert_equal 5, outgoing.size
+    outgoing.each { |issue| assert_equal target_urls, issue.details[:target_urls] }
+    missing = issues.find { |issue| issue.code == :indexable_page_missing_from_sitemap && issue.url.end_with?("/pricing") }
+    assert_equal source_urls.first, missing.details[:source_url]
+    assert_equal source_urls, missing.details[:source_urls]
+  end
+
+  def test_does_not_check_destination_canonicals_under_redirect_aliases
+    pages = ["old", "alias"].map do |path|
+      page(url: "https://example.com/#{path}", final_url: "https://example.com/guide", body: '<html><head><link rel="canonical" href="https://example.com/missing"></head></html>')
+    end
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: context)
+
+    canonical_issues = issues.select { |issue| [:canonical_no_internal_inlinks, :canonical_points_to_redirect, :canonical_points_to_error].include?(issue.code) }
+    assert_empty canonical_issues
+  end
+
   private
 
   def context(resolver: method(:resolve_target))
@@ -439,14 +475,14 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     }
   end
 
-  def page(url:, body:)
+  def page(url:, body:, final_url: url)
     doc = Nokogiri::HTML(body)
 
     Crawlscope::Page.new(
       url: url,
       normalized_url: url,
-      final_url: url,
-      normalized_final_url: url,
+      final_url: final_url,
+      normalized_final_url: final_url,
       status: 200,
       headers: {"content-type" => "text/html"},
       body: body,

data/test/crawlscope/metadata_rule_test.rb

@@ -80,13 +80,46 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     assert_includes codes, :multiple_title_tags
     assert_includes codes, :multiple_meta_descriptions
     assert_includes codes, :empty_h1
-    assert_includes codes, :canonical_mismatch
+    refute_includes codes, :canonical_mismatch
     assert_includes codes, :non_canonical_page_in_sitemap
+    mismatch = issues.find { |issue| issue.code == :non_canonical_page_in_sitemap }
+    assert_equal "sitemap lists this URL, but its canonical points to https://example.com/canonical-about", mismatch.message
+    assert_equal "https://example.com/canonical-about", mismatch.details[:canonical]
+  end
+
+  def test_length_messages_include_measurement_units_and_limit
+    issues = Crawlscope::IssueCollection.new
+    long_page = page(body: %(<html><head><title>#{"T" * 73}</title><meta name="description" content="#{"D" * 161}"></head></html>))
+
+    Crawlscope::Rules::Metadata.new.call(urls: [long_page.url], pages: [long_page], issues: issues)
+
+    assert_equal "title: 73 characters; maximum 72", issues.find { |issue| issue.code == :title_too_long }.message
+    assert_equal "meta description: 161 characters; maximum 160", issues.find { |issue| issue.code == :meta_description_too_long }.message
+  end
+
+  def test_does_not_apply_destination_metadata_to_redirect_alias
+    redirected = page(url: "https://example.com/old", final_url: "https://example.com/new", body: '<html><head><link rel="canonical" href="https://example.com/new"></head></html>')
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Metadata.new.call(urls: [redirected.url], pages: [redirected], issues: issues)
+
+    assert_empty issues.to_a
+  end
+
+  def test_keeps_canonical_mismatch_for_a_page_outside_the_sitemap
+    mismatched = page(body: '<html><head><link rel="canonical" href="https://example.com/preferred"></head></html>')
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Metadata.new.call(urls: [], pages: [mismatched], issues: issues)
+
+    canonical_issues = issues.select { |issue| [:canonical_mismatch, :non_canonical_page_in_sitemap].include?(issue.code) }
+    assert_equal [:canonical_mismatch], canonical_issues.map(&:code)
+    assert_equal "https://example.com/preferred", canonical_issues.first.details[:canonical]
   end
 
   private
 
-  def page(url: "https://example.com/about", body: nil)
+  def page(url: "https://example.com/about", body: nil, final_url: url)
     body ||= <<~HTML
       <html>
         <head>
@@ -102,8 +135,8 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     Crawlscope::Page.new(
       url: url,
       normalized_url: Crawlscope::Url.normalize(url, base_url: url),
-      final_url: url,
-      normalized_final_url: Crawlscope::Url.normalize(url, base_url: url),
+      final_url: final_url,
+      normalized_final_url: Crawlscope::Url.normalize(final_url, base_url: url),
       status: 200,
       headers: {"content-type" => "text/html"},
       body: body,