crawlscope 0.7.2 → 0.8.1

data/lib/crawlscope/request_headers.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Crawlscope
+  module RequestHeaders
+    PROFILE_TOKEN = "X-Profile-Token"
+
+    module_function
+
+    def add_profile_token(headers, url:, base_url:, profile_token:)
+      return headers if profile_token.to_s.empty?
+      return headers unless same_origin?(url, base_url: base_url)
+
+      headers[PROFILE_TOKEN] = profile_token
+      headers
+    end
+
+    def strip_profile_token_on_cross_origin_redirect(response_env, request_env)
+      return if same_origin?(request_env[:url], base_url: response_env[:url])
+
+      request_env[:request_headers].delete(PROFILE_TOKEN)
+    end
+
+    def same_origin?(url, base_url:)
+      url_uri = URI.join(base_url.to_s, url.to_s)
+      base_uri = URI.parse(base_url.to_s)
+
+      [url_uri.scheme, url_uri.host, url_uri.port] == [base_uri.scheme, base_uri.host, base_uri.port]
+    rescue URI::Error
+      false
+    end
+  end
+end

data/lib/crawlscope/result.rb

@@ -5,5 +5,9 @@ module Crawlscope
     def ok?
       issues.none?(&:error?)
     end
+
+    def server_timing_summary
+      ServerTiming::Summary.new(pages)
+    end
   end
 end

data/lib/crawlscope/rules/content_quality.rb

@@ -44,7 +44,7 @@ module Crawlscope
           severity: :warning,
           category: :content_quality,
           url: page.url,
-          message: "visible text has low token variety (#{format_ratio(ratio)})",
+          message: "unique words: #{format_percentage(ratio)}; minimum #{format_percentage(@min_unique_token_ratio)}",
           details: {
             ratio: ratio.round(3),
             threshold: @min_unique_token_ratio,
@@ -67,7 +67,7 @@ module Crawlscope
           severity: :warning,
           category: :content_quality,
           url: page.url,
-          message: "low visible text to HTML ratio (#{format_ratio(ratio)})",
+          message: "visible text / HTML bytes: #{format_percentage(ratio)}; minimum #{format_percentage(@min_visible_text_ratio)}",
           details: {
             html_bytes: html_bytes,
             ratio: ratio.round(3),
@@ -86,13 +86,13 @@ module Crawlscope
           severity: :warning,
           category: :content_quality,
           url: page.url,
-          message: "thin visible text (#{word_count} words)",
+          message: "visible words: #{word_count}; minimum #{@min_visible_words}",
           details: {word_count: word_count, minimum: @min_visible_words}
         )
       end
 
-      def format_ratio(value)
-        format("%.2f", value)
+      def format_percentage(value)
+        "#{format("%.3f", value * 100).sub(/0+\z/, "").sub(/\.\z/, "")}%"
       end
     end
   end

data/lib/crawlscope/rules/indexability.rb

@@ -192,7 +192,7 @@ module Crawlscope
           severity: :error,
           category: :sitemaps,
           url: page.url,
-          message: "sitemap URL is noindex",
+          message: "sitemap lists a URL that blocks indexing",
           details: {content: content, source: source}
         )
       end

data/lib/crawlscope/rules/links.rb

@@ -8,7 +8,6 @@ module Crawlscope
       LINK_SELECTORS = "a[href]"
       INTERNAL_PATH_PREFIXES_TO_SKIP = ["/rails/", "/cdn-cgi/"].freeze
       LINK_SCHEMES_TO_SKIP = ["mailto:", "tel:", "javascript:", "data:"].freeze
-      MAX_SOURCES_IN_ERROR = 3
       MIN_INBOUND_ANCHOR_LINKS = 1
       MIN_DOFOLLOW_INBOUND_LINKS = 2
 
@@ -128,20 +127,20 @@ module Crawlscope
       end
 
       def report_broken_target(target_url, grouped_links, issues, status)
-        source_urls = grouped_links.map { |link| link[:source_url] }.uniq.first(MAX_SOURCES_IN_ERROR)
+        source_urls = grouped_links.map { |link| link[:source_url] }.uniq
         issues.add(
           code: :broken_internal_link,
           severity: :warning,
           category: :links,
           url: target_url,
-          message: "broken internal link (HTTP #{status}, sources: #{source_urls.join(", ")})",
+          message: "linked URL returns HTTP #{status}",
           details: {source_urls: source_urls, status: status}
         )
       end
 
       def validate_nofollow_outgoing_links(links, issues)
         links.select { |link| link[:nofollow] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
-          target_urls = grouped_links.map { |link| link[:target_url] }.uniq.first(MAX_SOURCES_IN_ERROR)
+          target_urls = grouped_links.map { |link| link[:target_url] }.uniq
 
           issues.add(
             code: :nofollow_internal_outlinks,
@@ -156,7 +155,7 @@ module Crawlscope
 
       def validate_http_internal_links(links, issues)
         links.select { |link| link[:http_internal_link] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
-          target_urls = grouped_links.map { |link| link[:target_url] }.uniq.first(MAX_SOURCES_IN_ERROR)
+          target_urls = grouped_links.map { |link| link[:target_url] }.uniq
 
           issues.add(
             code: :http_internal_link,
@@ -170,15 +169,15 @@ module Crawlscope
       end
 
       def report_unresolved_target(target_url, grouped_links, issues, resolution)
-        source_urls = grouped_links.map { |link| link[:source_url] }.uniq.first(MAX_SOURCES_IN_ERROR)
-        suffix = (resolution && resolution[:error]) ? " (#{resolution[:error]})" : ""
+        source_urls = grouped_links.map { |link| link[:source_url] }.uniq
+        suffix = (resolution && resolution[:error]) ? ": #{resolution[:error]}" : ""
 
         issues.add(
           code: :unresolved_internal_link,
           severity: :warning,
           category: :links,
           url: target_url,
-          message: "unable to validate internal link#{suffix} (sources: #{source_urls.join(", ")})",
+          message: "cannot check linked URL#{suffix}",
           details: {error: resolution && resolution[:error], source_urls: source_urls}
         )
       end
@@ -218,13 +217,13 @@ module Crawlscope
       end
 
       def report_redirect_target(target_url, grouped_links, issues, target)
-        source_urls = grouped_links.map { |link| link[:source_url] }.uniq.first(MAX_SOURCES_IN_ERROR)
+        source_urls = grouped_links.map { |link| link[:source_url] }.uniq
         issues.add(
           code: :internal_link_redirects,
           severity: :warning,
           category: :links,
           url: target_url,
-          message: "internal link redirects to #{target.final_url} (sources: #{source_urls.join(", ")})",
+          message: "internal link redirects to #{target.final_url}",
           details: {final_url: target.final_url, source_urls: source_urls, status: target.status}
         )
       end
@@ -361,15 +360,14 @@ module Crawlscope
           report_orphan_page(target_url, issues) if inbound_count.zero?
 
           if inbound_count.positive? && inbound_count < MIN_INBOUND_ANCHOR_LINKS
-            source_samples = sample_sources_by_target[path].first(MAX_SOURCES_IN_ERROR)
-            source_info = source_samples.any? ? " (sources: #{source_samples.join(", ")})" : ""
+            source_samples = sample_sources_by_target[path]
 
             issues.add(
               code: :low_inbound_anchor_links,
               severity: :warning,
               category: :links,
               url: target_url,
-              message: "inbound anchor links #{inbound_count} below #{MIN_INBOUND_ANCHOR_LINKS}#{source_info}",
+              message: "incoming internal links: #{inbound_count}; minimum #{MIN_INBOUND_ANCHOR_LINKS}",
               details: {inbound_count: inbound_count, minimum: MIN_INBOUND_ANCHOR_LINKS, source_urls: source_samples}
             )
           end
@@ -414,7 +412,7 @@ module Crawlscope
           severity: :notice,
           category: :url,
           url: url,
-          message: "URL too long (#{url.length})",
+          message: "URL: #{url.length} characters; maximum 2048",
           details: {length: url.length, maximum: 2_048}
         )
       end
@@ -446,27 +444,22 @@ module Crawlscope
 
       def validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
         sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
-        reported_urls = Set.new
-
-        resolved_links.each do |link|
-          final_url = link[:final_url]
+        resolved_links.group_by { |link| link[:final_url] }.each do |final_url, grouped_links|
+          link = grouped_links.first
           next if sitemap_urls.include?(final_url)
-          next if reported_urls.include?(final_url)
           next unless crawlable_path?(link[:final_path])
 
           target = target_for(final_url)
           next unless target.allowed?(@allowed_statuses) && target.html?
           next if target.noindex?
 
-          reported_urls << final_url
-
           issues.add(
             code: :indexable_page_missing_from_sitemap,
             severity: :warning,
             category: :sitemaps,
             url: final_url,
-            message: "indexable internal page is missing from sitemap",
-            details: {source_url: link[:source_url]}
+            message: "linked page is missing from sitemap",
+            details: {source_url: link[:source_url], source_urls: grouped_links.map { |entry| entry[:source_url] }.uniq}
           )
         end
       end
@@ -486,15 +479,14 @@ module Crawlscope
         return if dofollow_count.zero?
         return if dofollow_count >= MIN_DOFOLLOW_INBOUND_LINKS
 
-        source_samples = sources_by_target[path].first(MAX_SOURCES_IN_ERROR)
-        source_info = source_samples.any? ? " (sources: #{source_samples.join(", ")})" : ""
+        source_samples = sources_by_target[path]
 
         issues.add(
           code: :low_dofollow_inlinks,
           severity: :warning,
           category: :links,
           url: target_url,
-          message: "dofollow inbound links #{dofollow_count} below #{MIN_DOFOLLOW_INBOUND_LINKS}#{source_info}",
+          message: "incoming internal links without nofollow: #{dofollow_count}; minimum #{MIN_DOFOLLOW_INBOUND_LINKS}",
           details: {dofollow_inbound_count: dofollow_count, minimum: MIN_DOFOLLOW_INBOUND_LINKS, source_urls: source_samples}
         )
       end
@@ -507,8 +499,8 @@ module Crawlscope
           severity: :warning,
           category: :links,
           url: target_url,
-          message: "page has nofollow incoming internal links only",
-          details: {nofollow_inbound_count: nofollow_count, source_urls: nofollow_sources.first(MAX_SOURCES_IN_ERROR)}
+          message: "all incoming internal links use nofollow",
+          details: {nofollow_inbound_count: nofollow_count, source_urls: nofollow_sources}
         )
       end
 
@@ -524,15 +516,17 @@ module Crawlscope
           details: {
             dofollow_inbound_count: dofollow_count,
             nofollow_inbound_count: nofollow_count,
-            dofollow_source_urls: dofollow_sources.first(MAX_SOURCES_IN_ERROR),
-            nofollow_source_urls: nofollow_sources.first(MAX_SOURCES_IN_ERROR)
+            dofollow_source_urls: dofollow_sources,
+            nofollow_source_urls: nofollow_sources
           }
         )
       end
 
       def validate_canonical_targets(urls, pages, resolved_links, issues)
         sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact
-        sitemap_pages = pages.select { |page| page.html? && sitemap_urls.include?(page.normalized_url) }
+        sitemap_pages = pages.select do |page|
+          page.html? && sitemap_urls.include?(page.normalized_url) && page.normalized_url.to_s == page.normalized_final_url.to_s
+        end
         return if sitemap_pages.size < 2
 
         dofollow_counts_by_path = dofollow_counts_by_final_path(resolved_links)
@@ -549,7 +543,7 @@ module Crawlscope
               severity: :warning,
               category: :links,
               url: canonical_url,
-              message: "canonical URL has no incoming internal links",
+              message: "canonical URL has no incoming internal links without nofollow",
               details: {source_url: page.url}
             )
           end
@@ -609,7 +603,7 @@ module Crawlscope
             severity: :warning,
             category: :metadata,
             url: page.url,
-            message: "canonical points to redirect",
+            message: "canonical #{canonical_url} redirects to #{target.final_url}",
             details: {canonical: canonical_url, final_url: target.final_url, status: target.status}
           )
         elsif !target.allowed?(@allowed_statuses)
@@ -618,7 +612,7 @@ module Crawlscope
             severity: :warning,
             category: :metadata,
             url: page.url,
-            message: "canonical points to HTTP #{target.status}",
+            message: "canonical #{canonical_url} returns HTTP #{target.status}",
             details: {canonical: canonical_url, status: target.status}
           )
         end

data/lib/crawlscope/rules/metadata.rb

@@ -21,7 +21,7 @@ module Crawlscope
         sitemap_urls = normalized_sitemap_urls(urls)
 
         pages.each do |page|
-          next unless page.html?
+          next unless page.html? && page.normalized_url.to_s == page.normalized_final_url.to_s
 
           validate_h1(page, issues)
           validate_title(page, issues)
@@ -47,7 +47,7 @@ module Crawlscope
             severity: :warning,
             category: :metadata,
             url: page.url,
-            message: "empty <h1>",
+            message: "empty <h1> tags: #{empty_h1s.size}",
             details: {count: empty_h1s.size}
           )
         end
@@ -93,7 +93,7 @@ module Crawlscope
         if title.empty?
           issues.add(code: :missing_title, severity: :warning, category: :metadata, url: page.url, message: "missing <title>", details: {})
         elsif title.length > TITLE_MAX_LENGTH
-          issues.add(code: :title_too_long, severity: :warning, category: :metadata, url: page.url, message: "title too long (#{title.length})", details: {length: title.length})
+          issues.add(code: :title_too_long, severity: :warning, category: :metadata, url: page.url, message: "title: #{title.length} characters; maximum #{TITLE_MAX_LENGTH}", details: {length: title.length})
         elsif repeated_site_name?(title)
           issues.add(code: :title_repeats_site_name, severity: :warning, category: :metadata, url: page.url, message: "title repeats #{@site_name}", details: {site_name: @site_name})
         end
@@ -117,9 +117,9 @@ module Crawlscope
         if description.empty?
           issues.add(code: :missing_meta_description, severity: :warning, category: :metadata, url: page.url, message: "missing meta description", details: {})
         elsif description.length < DESCRIPTION_MIN_LENGTH
-          issues.add(code: :meta_description_too_short, severity: :warning, category: :metadata, url: page.url, message: "meta description too short (#{description.length})", details: {length: description.length, minimum: DESCRIPTION_MIN_LENGTH})
+          issues.add(code: :meta_description_too_short, severity: :warning, category: :metadata, url: page.url, message: "meta description: #{description.length} characters; minimum #{DESCRIPTION_MIN_LENGTH}", details: {length: description.length, minimum: DESCRIPTION_MIN_LENGTH})
         elsif description.length > DESCRIPTION_MAX_LENGTH
-          issues.add(code: :meta_description_too_long, severity: :warning, category: :metadata, url: page.url, message: "meta description too long (#{description.length})", details: {length: description.length})
+          issues.add(code: :meta_description_too_long, severity: :warning, category: :metadata, url: page.url, message: "meta description: #{description.length} characters; maximum #{DESCRIPTION_MAX_LENGTH}", details: {length: description.length})
         end
       end
 
@@ -136,25 +136,25 @@ module Crawlscope
         return if canonical_matches_page?(normalized_canonical, normalized_page_url)
 
         details = {canonical: canonical}
-        issues.add(
-          code: :canonical_mismatch,
-          severity: :warning,
-          category: :metadata,
-          url: page.url,
-          message: "canonical mismatch (#{canonical})",
-          details: details
-        )
-
-        return unless sitemap_urls.include?(normalized_page_url)
-
-        issues.add(
-          code: :non_canonical_page_in_sitemap,
-          severity: :warning,
-          category: :sitemaps,
-          url: page.url,
-          message: "non-canonical page is included in sitemap",
-          details: details
-        )
+        if sitemap_urls.include?(normalized_page_url)
+          issues.add(
+            code: :non_canonical_page_in_sitemap,
+            severity: :warning,
+            category: :sitemaps,
+            url: page.url,
+            message: "sitemap lists this URL, but its canonical points to #{canonical}",
+            details: details
+          )
+        else
+          issues.add(
+            code: :canonical_mismatch,
+            severity: :warning,
+            category: :metadata,
+            url: page.url,
+            message: "canonical points to a different URL: #{canonical}",
+            details: details
+          )
+        end
       end
 
       def repeated_site_name?(title)
@@ -174,7 +174,7 @@ module Crawlscope
           severity: :warning,
           category: :metadata,
           url: page.url,
-          message: "Open Graph tags incomplete (missing #{missing.join(", ")})",
+          message: "missing Open Graph tags: #{missing.join(", ")}",
           details: {missing: missing}
         )
       end

data/lib/crawlscope/rules/structured_data.rb

@@ -49,7 +49,7 @@ module Crawlscope
               severity: :warning,
               category: :structured_data,
               url: page.url,
-              message: "#{source} parse error: #{data[:message]}",
+              message: "#{source_label(source)} parse error: #{data[:message]}",
               details: {source: source}
             )
             next
@@ -65,7 +65,7 @@ module Crawlscope
             severity: :warning,
             category: :structured_data,
             url: page.url,
-            message: "#{source} schema errors: #{errors.to_json}",
+            message: schema_error_message(source, errors),
             details: {errors: errors, source: source}
           )
         end
@@ -73,6 +73,15 @@ module Crawlscope
         validate_job_posting_count(page, items, issues)
       end
 
+      def source_label(source)
+        (source == "json-ld") ? "JSON-LD" : source
+      end
+
+      def schema_error_message(source, errors)
+        messages = errors.map { |error| Crawlscope::StructuredData::ErrorFormatter.format(error) }
+        "#{source_label(source)} #{messages.join("; ")}"
+      end
+
       def validate_job_posting_count(page, items, issues)
         job_postings = items.select { |item| structured_data_types(item.data).include?("JobPosting") }
         return if job_postings.size == 1
@@ -83,7 +92,7 @@ module Crawlscope
             severity: :warning,
             category: :structured_data,
             url: page.url,
-            message: "multiple JobPosting structured data blocks found",
+            message: "JobPosting blocks: #{job_postings.size}; expected 1",
             details: {count: job_postings.size}
           )
         elsif career_detail_page?(page.url)
@@ -107,7 +116,7 @@ module Crawlscope
           severity: :warning,
           category: :structured_data,
           url: page.url,
-          message: "#{source} structured data missing @type",
+          message: "#{source_label(source)} structured data missing @type",
           details: {paths: missing_paths, source: source}
         )
       end

data/lib/crawlscope/rules/uniqueness.rb

@@ -66,7 +66,9 @@ module Crawlscope
       end
 
       def summarize_pages(pages)
-        html_pages = pages.select(&:html?)
+        html_pages = pages.select do |page|
+          page.html? && page.normalized_url.to_s == page.normalized_final_url.to_s
+        end
 
         FetchExecutor.map(
           name: @fetch_executor,
@@ -82,7 +84,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate title '#{value}' => #{urls.join(", ")}",
+            message: "same title: #{value.inspect}",
             details: {urls: urls, value: value}
           )
         end
@@ -93,7 +95,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate meta description '#{value}' => #{urls.join(", ")}",
+            message: "same meta description: #{value.inspect}",
             details: {urls: urls, value: value}
           )
         end
@@ -104,7 +106,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate page content fingerprint => #{urls.join(", ")}",
+            message: "same page text",
             details: {urls: urls}
           )
         end
@@ -115,7 +117,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "duplicate pages without canonical => #{urls.join(", ")}",
+            message: "same page text; at least one page has no canonical link",
             details: {urls: urls}
           )
         end
@@ -144,7 +146,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "near duplicate scan skipped for #{page_summaries.size} pages",
+            message: "similar-page check skipped: #{page_summaries.size} pages; limit #{@max_near_duplicate_pages}",
             details: {max_pages: @max_near_duplicate_pages, page_count: page_summaries.size}
           )
           return
@@ -164,7 +166,7 @@ module Crawlscope
             severity: :warning,
             category: :uniqueness,
             url: nil,
-            message: "near duplicate page content (#{format("%.2f", similarity)}) => #{urls.join(", ")}",
+            message: "text similarity: #{format("%.1f", similarity * 100)}%; threshold #{format("%.1f", @near_duplicate_threshold * 100)}%",
             details: {similarity: similarity.round(3), threshold: @near_duplicate_threshold, urls: urls}
           )
         end

data/lib/crawlscope/run.rb

@@ -44,17 +44,7 @@ module Crawlscope
       value = @configuration.sitemap_path
       return value unless value.to_s.strip.empty?
 
-      local_path = File.expand_path("public/sitemap.xml", Dir.pwd)
-      return local_path if local_path_default?(base_url: base_url) && File.exist?(local_path)
-
       "#{base_url.to_s.chomp("/")}/sitemap.xml"
     end
-
-    def local_path_default?(base_url:)
-      host = URI.parse(base_url.to_s).host.to_s
-      ["localhost", "127.0.0.1"].include?(host)
-    rescue URI::InvalidURIError
-      false
-    end
   end
 end

data/lib/crawlscope/schema_registry.rb

@@ -51,11 +51,16 @@ module Crawlscope
       return errors if schema.nil?
 
       JSON::Validator.fully_validate(schema, item, errors_as_objects: true).each do |error|
-        errors << {
+        details = {
           field: error[:fragment].to_s.sub("#/", ""),
           issue: error[:message],
           type: type
         }
+        if %w[MaxLength MinLength].include?(error[:failed_attribute])
+          value = value_at_fragment(item, error[:fragment])
+          details[:length] = value.length if value.is_a?(String)
+        end
+        errors << details
       end
 
       errors
@@ -69,6 +74,17 @@ module Crawlscope
 
     private
 
+    def value_at_fragment(item, fragment)
+      fragment.to_s.delete_prefix("#/").split("/").reduce(item) do |value, key|
+        case value
+        when Hash
+          value[key]
+        when Array
+          value[key.to_i] if key.match?(/\A\d+\z/)
+        end
+      end
+    end
+
     def deep_copy(value)
       case value
       when Hash