crawlscope 0.8.0 → 0.9.0

data/lib/crawlscope/rules/uniqueness.rb

@@ -66,7 +66,9 @@ module Crawlscope
       end
 
       def summarize_pages(pages)
-        html_pages = pages.select(&:html?)
+        html_pages = pages.select do |page|
+          page.html? && page.normalized_url.to_s == page.normalized_final_url.to_s
+        end
 
         FetchExecutor.map(
           name: @fetch_executor,
@@ -82,7 +84,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate title '#{value}' => #{urls.join(", ")}",
+            message: "same title: #{value.inspect}",
             details: {urls: urls, value: value}
           )
         end
@@ -93,7 +95,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate meta description '#{value}' => #{urls.join(", ")}",
+            message: "same meta description: #{value.inspect}",
             details: {urls: urls, value: value}
           )
         end
@@ -104,7 +106,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate page content fingerprint => #{urls.join(", ")}",
+            message: "same page text",
             details: {urls: urls}
           )
         end
@@ -115,7 +117,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate pages without canonical => #{urls.join(", ")}",
+            message: "same page text; at least one page has no canonical link",
             details: {urls: urls}
           )
         end
@@ -144,7 +146,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "near duplicate scan skipped for #{page_summaries.size} pages",
+            message: "similar-page check skipped: #{page_summaries.size} pages; limit #{@max_near_duplicate_pages}",
             details: {max_pages: @max_near_duplicate_pages, page_count: page_summaries.size}
           )
           return
@@ -164,7 +166,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "near duplicate page content (#{format("%.2f", similarity)}) => #{urls.join(", ")}",
+            message: "text similarity: #{format("%.1f", similarity * 100)}%; threshold #{format("%.1f", @near_duplicate_threshold * 100)}%",
             details: {similarity: similarity.round(3), threshold: @near_duplicate_threshold, urls: urls}
           )
         end

data/lib/crawlscope/schema_registry.rb

@@ -51,11 +51,16 @@ module Crawlscope
       return errors if schema.nil?
 
       JSON::Validator.fully_validate(schema, item, errors_as_objects: true).each do |error|
-        errors << {
+        details = {
           field: error[:fragment].to_s.sub("#/", ""),
           issue: error[:message],
           type: type
         }
+        if %w[MaxLength MinLength].include?(error[:failed_attribute])
+          value = value_at_fragment(item, error[:fragment])
+          details[:length] = value.length if value.is_a?(String)
+        end
+        errors << details
       end
 
       errors
@@ -69,6 +74,17 @@ module Crawlscope
 
     private
 
+    def value_at_fragment(item, fragment)
+      fragment.to_s.delete_prefix("#/").split("/").reduce(item) do |value, key|
+        case value
+        when Hash
+          value[key]
+        when Array
+          value[key.to_i] if key.match?(/\A\d+\z/)
+        end
+      end
+    end
+
     def deep_copy(value)
       case value
       when Hash

data/lib/crawlscope/server_timing/reporter.rb

@@ -5,6 +5,8 @@ require "uri"
 module Crawlscope
   class ServerTiming
     class Reporter
+      SLOW_PAGE_THRESHOLD_MS = 50
+
       def initialize(io:)
         @io = io
       end
@@ -22,7 +24,7 @@ module Crawlscope
 
         report_metrics(summary.metrics)
         report_signals(summary.signals)
-        report_worst_pages(summary.worst_pages, base_url: base_url)
+        report_worst_pages(summary.worst_pages(limit: summary.pages.size), base_url: base_url)
         @io.puts("  Ignored malformed entries: #{summary.invalid_count}") if summary.invalid_count.positive?
       end
 
@@ -31,7 +33,7 @@ module Crawlscope
       def report_metrics(metrics)
         return if metrics.empty?
 
-        @io.puts("  Duration metrics (dur; milliseconds assumed):")
+        @io.puts("  Response times (ms):")
         metrics.each do |metric|
           label = metric[:name]
           label += " #{description(metric[:description])}" if metric[:description]
@@ -65,9 +67,10 @@ module Crawlscope
       end
 
       def report_worst_pages(samples, base_url:)
+        samples = samples.select { |sample| sample[:metric].duration > SLOW_PAGE_THRESHOLD_MS }
         return if samples.empty?
 
-        @io.puts("  Worst pages:")
+        @io.puts("  Pages above #{SLOW_PAGE_THRESHOLD_MS} ms:")
         samples.each do |sample|
           metric = sample[:metric]
           detail = metric.description ? " #{description(metric.description)}" : ""
@@ -84,7 +87,6 @@ module Crawlscope
 
       def description(value)
         value = value.to_s
-        value = "#{value[0, 77]}..." if value.length > 80
         value.inspect
       end
 

data/lib/crawlscope/structured_data/error_formatter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  module StructuredData
+    class ErrorFormatter
+      def self.format(error)
+        field = (error[:field] || error["field"]).to_s.sub(/\A#\/?/, "")
+        type = error[:type] || error["type"]
+        issue = error[:issue] || error["issue"] || "Unknown error"
+
+        if (match = issue.match(/did not contain a required property of '([^']+)'/))
+          field = [field, match[1]].reject(&:empty?).join("/")
+          message = "is required"
+        else
+          message = concise_message(issue, error[:length] || error["length"])
+        end
+
+        location = [type, field.empty? ? nil : field].compact.join(".")
+        "#{location.empty? ? "$" : location}: #{message}"
+      end
+
+      def self.concise_message(issue, length)
+        if length
+          unit = (length == 1) ? "character" : "characters"
+          measured_length = "#{length} #{unit}"
+        end
+        case issue
+        when /was not of a maximum string length of (\d+)/
+          length ? "#{measured_length}; maximum #{Regexp.last_match(1)}" : "exceeds #{Regexp.last_match(1)} characters"
+        when /was not of a minimum string length of (\d+)/
+          length ? "#{measured_length}; minimum #{Regexp.last_match(1)}" : "must have at least #{Regexp.last_match(1)} characters"
+        when /of type (\w+) did not match the following type: (\w+)/
+          "must be #{Regexp.last_match(2)} (got #{Regexp.last_match(1)})"
+        else
+          issue.sub(/\AThe property '[^']*' /, "")
+            .sub(/ in schema [\h-]{36}(?:#\S*)?\z/, "")
+        end
+      end
+      private_class_method :concise_message
+    end
+  end
+end

data/lib/crawlscope/structured_data/reporter.rb

@@ -67,10 +67,9 @@ module Crawlscope
           @io.puts("• #{extract_path(entry.url)}")
 
           entry.errors.each do |error|
+            @io.puts("    #{source_label(error[:source])}:")
             error[:errors].each do |validation_error|
-              field = validation_error[:field] || validation_error["field"] || "$"
-              issue = validation_error[:issue] || validation_error["issue"] || "Unknown error"
-              @io.puts("    - #{field}: #{issue}")
+              @io.puts("      - #{ErrorFormatter.format(validation_error)}")
             end
           end
         end
@@ -83,9 +82,7 @@ module Crawlscope
       end
 
       def report_entry(entry, debug:)
-        @io.puts("=" * 80)
         @io.puts("URL: #{entry.url}")
-        @io.puts("=" * 80)
 
         if entry.fetch_error
           @io.puts("Error: #{entry.fetch_error}")
@@ -124,19 +121,22 @@ module Crawlscope
 
       def report_validation(entry)
         @io.puts("")
-        @io.puts("Validation results:")
-
         if entry.errors.empty?
-          @io.puts("  All valid!")
+          @io.puts("Validation: OK")
         else
+          @io.puts("Validation: FAILED")
           entry.errors.each do |error|
-            @io.puts("  #{error[:type]}: INVALID [#{error[:source]}]")
+            @io.puts("  #{source_label(error[:source])}:")
             error[:errors].each do |validation_error|
-              @io.puts("    - field: #{validation_error[:field]}, issue: #{validation_error[:issue]}")
+              @io.puts("    - #{ErrorFormatter.format(validation_error)}")
             end
           end
         end
       end
+
+      def source_label(source)
+        (source == "json-ld") ? "JSON-LD" : source
+      end
     end
   end
 end

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

data/lib/crawlscope.rb

@@ -29,7 +29,7 @@ end
 Crawlscope.instance_variable_set(:@loader, Zeitwerk::Loader.for_gem)
 Crawlscope.loader.ignore("#{__dir__}/generators")
 Crawlscope.loader.ignore("#{__dir__}/tasks")
-Crawlscope.loader.ignore("#{__dir__}/crawlscope/railtie.rb")
+Crawlscope.loader.ignore("#{__dir__}/crawlscope/railtie.rb", "#{__dir__}/crawlscope/rails.rb")
 Crawlscope.loader.ignore("#{__dir__}/crawlscope/tasks.rb")
 Crawlscope.loader.setup
 

data/test/crawlscope/crawl_test.rb

@@ -325,6 +325,42 @@ class CrawlscopeCrawlTest < Minitest::Test
     assert_includes result.issues.to_a.map(&:code), :sitemap_redirect_url
   end
 
+  def test_reports_redirects_without_duplicate_destination_content
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/old</loc></url>
+          <url><loc>https://example.com/alias</loc></url>
+          <url><loc>https://example.com/new</loc></url>
+        </urlset>
+      XML
+    )
+    ["old", "alias"].each do |path|
+      stub_request(:get, "https://example.com/#{path}")
+        .to_return(status: 301, headers: {"Location" => "https://example.com/new"}, body: "")
+    end
+    body = %(<html><head><title>Current page</title><meta name="description" content="Current description"><link rel="canonical" href="https://example.com/new"></head><body>#{"Useful content " * 30}</body></html>)
+    stub_request(:get, "https://example.com/new")
+      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: body)
+
+    result = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [Crawlscope::Rules::Uniqueness.new, Crawlscope::Rules::Metadata.new],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
+    ).call
+
+    assert_equal 3, result.pages.size
+    assert_equal ["https://example.com/old", "https://example.com/alias"], result.issues.select { |issue| issue.code == :sitemap_redirect_url }.map(&:url)
+    refute_includes result.issues.to_a.map(&:code), :redirected_page
+    assert_empty result.issues.select { |issue| issue.category == :uniqueness }
+    assert_empty result.issues.select { |issue| [:canonical_mismatch, :non_canonical_page_in_sitemap].include?(issue.code) }
+    assert_equal 1, result.issues.count { |issue| issue.code == :meta_description_too_short }
+  end
+
   def test_resolves_uncrawled_link_targets_as_a_bounded_batch
     File.write(
       @sitemap_path,

data/test/crawlscope/links_rule_test.rb

@@ -56,7 +56,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     )
 
     assert_includes issues.to_a.map(&:code), :unresolved_internal_link
-    assert_includes issues.to_a.find { |issue| issue.code == :unresolved_internal_link }.message, "unable to validate internal link"
+    assert_includes issues.to_a.find { |issue| issue.code == :unresolved_internal_link }.message, "cannot check linked URL"
   end
 
   def test_ignores_fetch_errors_for_urls_already_crawled
@@ -429,6 +429,42 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     assert_empty issues.to_a
   end
 
+  def test_retains_all_link_sources_and_targets_in_issue_details
+    source_urls = 5.times.map { |index| "https://example.com/source-#{index}" }
+    target_urls = 5.times.map { |index| "https://example.com/target-#{index}" }
+    pages = source_urls.map do |url|
+      targets = target_urls.map { |target| %(<a href="#{target}" rel="nofollow">Target</a>) }.join
+      page(url: url, body: %(<html><body>#{targets}<a href="/missing">Missing</a><a href="/pricing">Pricing</a></body></html>))
+    end
+    resolver = lambda do |url|
+      {crawled: false, error: nil, final_url: url, html: true, status: url.end_with?("/missing") ? 404 : 200}
+    end
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(urls: source_urls, pages: pages, issues: issues, context: context(resolver: resolver))
+
+    broken = issues.find { |issue| issue.code == :broken_internal_link }
+    assert_equal source_urls, broken.details[:source_urls]
+    outgoing = issues.select { |issue| issue.code == :nofollow_internal_outlinks }
+    assert_equal 5, outgoing.size
+    outgoing.each { |issue| assert_equal target_urls, issue.details[:target_urls] }
+    missing = issues.find { |issue| issue.code == :indexable_page_missing_from_sitemap && issue.url.end_with?("/pricing") }
+    assert_equal source_urls.first, missing.details[:source_url]
+    assert_equal source_urls, missing.details[:source_urls]
+  end
+
+  def test_does_not_check_destination_canonicals_under_redirect_aliases
+    pages = ["old", "alias"].map do |path|
+      page(url: "https://example.com/#{path}", final_url: "https://example.com/guide", body: '<html><head><link rel="canonical" href="https://example.com/missing"></head></html>')
+    end
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Links.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: context)
+
+    canonical_issues = issues.select { |issue| [:canonical_no_internal_inlinks, :canonical_points_to_redirect, :canonical_points_to_error].include?(issue.code) }
+    assert_empty canonical_issues
+  end
+
   private
 
   def context(resolver: method(:resolve_target))
@@ -439,14 +475,14 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     }
   end
 
-  def page(url:, body:)
+  def page(url:, body:, final_url: url)
     doc = Nokogiri::HTML(body)
 
     Crawlscope::Page.new(
       url: url,
       normalized_url: url,
-      final_url: url,
-      normalized_final_url: url,
+      final_url: final_url,
+      normalized_final_url: final_url,
       status: 200,
       headers: {"content-type" => "text/html"},
       body: body,

data/test/crawlscope/loader_test.rb

@@ -15,6 +15,7 @@ class CrawlscopeLoaderTest < Minitest::Test
     script = <<~RUBY
       require "crawlscope"
 
+      abort "Rails loaded by the core" if defined?(Rails)
       loaded = $LOADED_FEATURES.grep(%r{/(?:async|faraday|json-schema|nokogiri)(?:/|\\.rb)})
       abort loaded.join("\\n") unless loaded.empty?
     RUBY
@@ -29,6 +30,30 @@ class CrawlscopeLoaderTest < Minitest::Test
     assert status.success?, stderr
   end
 
+  def test_explicit_rails_integration_loads_after_the_core_and_registers_tasks
+    script = <<~RUBY
+      require "crawlscope"
+      require "crawlscope/rails"
+
+      abort "Railtie missing" unless Crawlscope::Railtie < Rails::Railtie
+      app = Class.new(Rails::Application)
+      app.config.eager_load = false
+      app.load_tasks
+      %w[validate validate:ldjson validate:indexability validate:content_quality].each do |task|
+        abort "task missing: \#{task}" unless Rake::Task.task_defined?("crawlscope:\#{task}")
+      end
+    RUBY
+
+    _stdout, stderr, status = Open3.capture3(
+      RbConfig.ruby,
+      "-I#{File.expand_path("../../lib", __dir__)}",
+      "-e",
+      script
+    )
+
+    assert status.success?, stderr
+  end
+
   def test_task_entrypoint_registers_tasks
     script = <<~RUBY
       require "crawlscope/tasks"

data/test/crawlscope/metadata_rule_test.rb

@@ -80,13 +80,46 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     assert_includes codes, :multiple_title_tags
     assert_includes codes, :multiple_meta_descriptions
     assert_includes codes, :empty_h1
-    assert_includes codes, :canonical_mismatch
+    refute_includes codes, :canonical_mismatch
     assert_includes codes, :non_canonical_page_in_sitemap
+    mismatch = issues.find { |issue| issue.code == :non_canonical_page_in_sitemap }
+    assert_equal "sitemap lists this URL, but its canonical points to https://example.com/canonical-about", mismatch.message
+    assert_equal "https://example.com/canonical-about", mismatch.details[:canonical]
+  end
+
+  def test_length_messages_include_measurement_units_and_limit
+    issues = Crawlscope::IssueCollection.new
+    long_page = page(body: %(<html><head><title>#{"T" * 73}</title><meta name="description" content="#{"D" * 161}"></head></html>))
+
+    Crawlscope::Rules::Metadata.new.call(urls: [long_page.url], pages: [long_page], issues: issues)
+
+    assert_equal "title: 73 characters; maximum 72", issues.find { |issue| issue.code == :title_too_long }.message
+    assert_equal "meta description: 161 characters; maximum 160", issues.find { |issue| issue.code == :meta_description_too_long }.message
+  end
+
+  def test_does_not_apply_destination_metadata_to_redirect_alias
+    redirected = page(url: "https://example.com/old", final_url: "https://example.com/new", body: '<html><head><link rel="canonical" href="https://example.com/new"></head></html>')
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Metadata.new.call(urls: [redirected.url], pages: [redirected], issues: issues)
+
+    assert_empty issues.to_a
+  end
+
+  def test_keeps_canonical_mismatch_for_a_page_outside_the_sitemap
+    mismatched = page(body: '<html><head><link rel="canonical" href="https://example.com/preferred"></head></html>')
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Metadata.new.call(urls: [], pages: [mismatched], issues: issues)
+
+    canonical_issues = issues.select { |issue| [:canonical_mismatch, :non_canonical_page_in_sitemap].include?(issue.code) }
+    assert_equal [:canonical_mismatch], canonical_issues.map(&:code)
+    assert_equal "https://example.com/preferred", canonical_issues.first.details[:canonical]
   end
 
   private
 
-  def page(url: "https://example.com/about", body: nil)
+  def page(url: "https://example.com/about", body: nil, final_url: url)
     body ||= <<~HTML
       <html>
         <head>
@@ -102,8 +135,8 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     Crawlscope::Page.new(
       url: url,
       normalized_url: Crawlscope::Url.normalize(url, base_url: url),
-      final_url: url,
-      normalized_final_url: Crawlscope::Url.normalize(url, base_url: url),
+      final_url: final_url,
+      normalized_final_url: Crawlscope::Url.normalize(final_url, base_url: url),
       status: 200,
       headers: {"content-type" => "text/html"},
       body: body,