crawlscope 0.4.0 → 0.6.0

data/lib/crawlscope/reporter.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
+require "uri"
+
 module Crawlscope
   class Reporter
+    MAX_ISSUES_PER_GROUP = 20
+
     def initialize(io:)
       @io = io
     end
@@ -13,21 +17,141 @@ module Crawlscope
       @io.puts("URLs: #{result.urls.size}")
       @io.puts("Pages: #{result.pages.size}")
 
-      if result.ok?
+      if result.issues.size.zero?
         @io.puts("Status: OK")
         return
       end
 
-      @io.puts("Status: FAILED")
-      @io.puts("Issues: #{result.issues.size}")
+      @io.puts("Status: #{status_for(result.issues)}")
+      @io.puts("Issues: #{result.issues.size} total (#{severity_summary(result.issues)})")
+      @io.puts("")
+
+      report_summary(result.issues)
+      @io.puts("")
+      report_issue_groups(result.issues, base_url: result.base_url)
+    end
+
+    private
+
+    def status_for(issues)
+      grouped = issues.by_severity
 
-      result.issues.by_severity.sort_by { |severity, _issues| severity.to_s }.each do |severity, issues|
-        @io.puts("#{severity}: #{issues.size}")
+      if grouped.key?(:error)
+        "FAILED"
+      elsif grouped.key?(:warning)
+        "WARNINGS"
+      else
+        "NOTICES"
       end
+    end
+
+    def severity_summary(issues)
+      grouped = issues.by_severity
+      return "" if grouped.empty?
+
+      grouped
+        .sort_by { |severity, severity_issues| [-severity_issues.size, severity.to_s] }
+        .map { |severity, severity_issues| "#{severity_issues.size} #{pluralize(severity, severity_issues.size)}" }
+        .join(", ")
+    end
+
+    def report_summary(issues)
+      @io.puts("Summary:")
 
-      result.issues.each do |issue|
-        @io.puts("- [#{issue.severity}] #{issue.url} #{issue.message}")
+      issues.by_category
+        .sort_by { |category, category_issues| [-category_issues.size, category.to_s] }
+        .each do |category, category_issues|
+          @io.puts("  #{category.to_s.ljust(16)} #{category_issues.size}")
+        end
+    end
+
+    def report_issue_groups(issues, base_url:)
+      grouped = issues.to_a.group_by { |issue| [issue.category, issue.code] }
+
+      grouped
+        .sort_by { |(category, code), grouped_issues| [-grouped_issues.size, category.to_s, code.to_s] }
+        .each do |(category, code), grouped_issues|
+          @io.puts("#{category} / #{code}: #{grouped_issues.size}")
+
+          grouped_issues.first(MAX_ISSUES_PER_GROUP).each do |issue|
+            @io.puts("  - #{compact_issue(issue, base_url: base_url)}")
+          end
+
+          remaining_count = grouped_issues.size - MAX_ISSUES_PER_GROUP
+          @io.puts("  ... #{remaining_count} more") if remaining_count.positive?
+          @io.puts("")
+        end
+    end
+
+    def compact_issue(issue, base_url:)
+      parts = []
+      parts << relative_url(issue.url, base_url: base_url) if issue.url
+
+      detail = compact_detail(issue, base_url: base_url)
+      parts << detail unless detail.empty?
+
+      parts.compact.join("  ")
+    end
+
+    def compact_detail(issue, base_url:)
+      details = issue.details || {}
+      fragments = []
+
+      inbound = details[:dofollow_inbound_count] || details[:inbound_count]
+      fragments << "inbound #{inbound}/#{details[:minimum]}" if inbound && details[:minimum]
+
+      if details[:ratio] && details[:threshold]
+        fragments << "ratio #{format_number(details[:ratio])}/#{format_number(details[:threshold])}"
+      end
+
+      fragments << "count #{details[:count]}" if details[:count]
+      fragments << "length #{details[:length]}" if details[:length]
+      fragments << "status #{details[:status]}" if details[:status]
+      fragments << "final: #{relative_url(details[:final_url], base_url: base_url)}" if details[:final_url]
+      fragments << "sources: #{relative_urls(details[:source_urls], base_url: base_url).join(", ")}" if details[:source_urls]&.any?
+      fragments << "source: #{relative_url(details[:source_url], base_url: base_url)}" if details[:source_url]
+      fragments << "targets: #{relative_urls(details[:target_urls], base_url: base_url).join(", ")}" if details[:target_urls]&.any?
+
+      return issue.message if fragments.empty?
+
+      case issue.code
+      when :low_dofollow_inlinks, :low_inbound_anchor_links, :low_unique_token_ratio, :low_visible_text_ratio
+        fragments.join("  ")
+      else
+        ([issue.message] + fragments).join("  ")
       end
     end
+
+    def relative_urls(urls, base_url:)
+      Array(urls).map { |url| relative_url(url, base_url: base_url) }
+    end
+
+    def relative_url(url, base_url:)
+      return url unless url && base_url
+
+      uri = URI.parse(url)
+      base_uri = URI.parse(base_url)
+
+      return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
+
+      relative = uri.path.to_s.empty? ? "/" : uri.path
+      relative += "?#{uri.query}" if uri.query
+      relative += "##{uri.fragment}" if uri.fragment
+      relative
+    rescue URI::InvalidURIError
+      url
+    end
+
+    def format_number(value)
+      return format("%.3f", value) if value.is_a?(Float)
+
+      value.to_s
+    end
+
+    def pluralize(word, count)
+      return word.to_s if count == 1
+
+      "#{word}s"
+    end
   end
 end

data/lib/crawlscope/result.rb

@@ -3,7 +3,7 @@
 module Crawlscope
   Result = Data.define(:base_url, :sitemap_path, :urls, :pages, :issues) do
     def ok?
-      issues.none?(&:error?) && issues.none?(&:warning?) && issues.none?(&:notice?)
+      issues.none?(&:error?)
     end
   end
 end

data/lib/crawlscope/rules/content_quality.rb

@@ -55,7 +55,7 @@ module Crawlscope
       end
 
       def validate_visible_text_ratio(page, issues)
-        html_bytes = DocumentText.html_for(page.doc).bytesize
+        html_bytes = DocumentText.content_ratio_html_for(page.doc).bytesize
         return if html_bytes.zero?
 
         visible_text = DocumentText.text_for(page.doc)

data/lib/crawlscope/rules/indexability.rb

@@ -6,6 +6,31 @@ module Crawlscope
       ROBOTS_META_SELECTOR = 'meta[name="robots"], meta[name="googlebot"]'
       X_ROBOTS_TAG_HEADER = "x-robots-tag"
 
+      def self.noindex_header?(headers)
+        noindex?(header_value(headers, X_ROBOTS_TAG_HEADER))
+      end
+
+      def self.noindex_meta?(doc)
+        return false unless doc
+
+        doc.css(ROBOTS_META_SELECTOR).any? { |tag| noindex?(tag["content"].to_s) }
+      end
+
+      def self.header_value(headers, name)
+        headers.find { |key, _value| key.to_s.casecmp?(name) }&.last.to_s
+      end
+
+      def self.directives(value)
+        value
+          .split(",")
+          .map { |directive| directive.split(":", 2).last.to_s.strip }
+          .reject(&:empty?)
+      end
+
+      def self.noindex?(value)
+        directives(value).any? { |directive| directive.casecmp?("noindex") || directive.casecmp?("none") }
+      end
+
       attr_reader :code
 
       def initialize
@@ -13,45 +38,107 @@ module Crawlscope
       end
 
       def call(urls:, pages:, issues:, context: nil)
+        sitemap_urls = normalized_sitemap_urls(urls)
+
         pages.each do |page|
-          validate_meta_robots(page, issues) if page.html?
-          validate_x_robots_tag(page, issues)
+          validate_meta_robots(page, issues, sitemap_urls) if page.html?
+          validate_x_robots_tag(page, issues, sitemap_urls)
         end
       end
 
       private
 
+      def normalized_sitemap_urls(urls)
+        urls.map { |url| Url.normalize(url, base_url: url) }.compact
+      end
+
       def header_value(page, name)
-        page.headers.find { |key, _value| key.to_s.casecmp?(name) }&.last.to_s
+        self.class.header_value(page.headers, name)
+      end
+
+      def directives(value)
+        self.class.directives(value)
       end
 
       def noindex?(value)
-        value
-          .split(",")
-          .map { |directive| directive.split(":", 2).last.to_s.strip }
-          .any? { |directive| directive.casecmp?("noindex") || directive.casecmp?("none") }
+        self.class.noindex?(value)
+      end
+
+      def follow?(value)
+        directives(value).any? { |directive| directive.casecmp?("follow") }
       end
 
-      def validate_meta_robots(page, issues)
+      def nofollow?(value)
+        directives(value).any? { |directive| directive.casecmp?("nofollow") || directive.casecmp?("none") }
+      end
+
+      def validate_meta_robots(page, issues, sitemap_urls)
         page.doc.css(ROBOTS_META_SELECTOR).each do |tag|
           content = tag["content"].to_s
-          next unless noindex?(content)
-
-          issues.add(
-            code: :noindex_meta,
-            severity: :error,
-            category: :indexability,
-            url: page.url,
-            message: "robots meta tag prevents indexing",
-            details: {content: content, name: tag["name"].to_s}
-          )
+
+          report_noindex_meta(page, issues, content, tag["name"].to_s, sitemap_urls) if noindex?(content)
+          report_nofollow_meta(page, issues, content, tag["name"].to_s) if nofollow?(content)
+          report_noindex_follow_meta(page, issues, content, tag["name"].to_s) if noindex?(content) && follow?(content)
+          report_noindex_nofollow_meta(page, issues, content, tag["name"].to_s) if noindex?(content) && nofollow?(content)
         end
       end
 
-      def validate_x_robots_tag(page, issues)
+      def validate_x_robots_tag(page, issues, sitemap_urls)
         content = header_value(page, X_ROBOTS_TAG_HEADER)
-        return unless noindex?(content)
+        return if content.empty?
+
+        report_noindex_header(page, issues, content, sitemap_urls) if noindex?(content)
+        report_nofollow_header(page, issues, content) if nofollow?(content)
+        report_noindex_follow_header(page, issues, content) if noindex?(content) && follow?(content)
+        report_noindex_nofollow_header(page, issues, content) if noindex?(content) && nofollow?(content)
+      end
+
+      def report_noindex_meta(page, issues, content, name, sitemap_urls)
+        issues.add(
+          code: :noindex_meta,
+          severity: :error,
+          category: :indexability,
+          url: page.url,
+          message: "robots meta tag prevents indexing",
+          details: {content: content, name: name}
+        )
+        report_sitemap_noindex_url(page, issues, content, source: "meta", sitemap_urls: sitemap_urls)
+      end
+
+      def report_nofollow_meta(page, issues, content, name)
+        issues.add(
+          code: :nofollow_meta,
+          severity: :warning,
+          category: :indexability,
+          url: page.url,
+          message: "robots meta tag prevents following links",
+          details: {content: content, name: name}
+        )
+      end
 
+      def report_noindex_follow_meta(page, issues, content, name)
+        issues.add(
+          code: :noindex_follow_meta,
+          severity: :warning,
+          category: :indexability,
+          url: page.url,
+          message: "robots meta tag prevents indexing but allows following links",
+          details: {content: content, name: name}
+        )
+      end
+
+      def report_noindex_nofollow_meta(page, issues, content, name)
+        issues.add(
+          code: :noindex_nofollow_meta,
+          severity: :error,
+          category: :indexability,
+          url: page.url,
+          message: "robots meta tag prevents indexing and following links",
+          details: {content: content, name: name}
+        )
+      end
+
+      def report_noindex_header(page, issues, content, sitemap_urls)
         issues.add(
           code: :noindex_header,
           severity: :error,
@@ -60,6 +147,54 @@ module Crawlscope
           message: "X-Robots-Tag header prevents indexing",
           details: {content: content}
         )
+        report_sitemap_noindex_url(page, issues, content, source: "header", sitemap_urls: sitemap_urls)
+      end
+
+      def report_nofollow_header(page, issues, content)
+        issues.add(
+          code: :nofollow_header,
+          severity: :warning,
+          category: :indexability,
+          url: page.url,
+          message: "X-Robots-Tag header prevents following links",
+          details: {content: content}
+        )
+      end
+
+      def report_noindex_follow_header(page, issues, content)
+        issues.add(
+          code: :noindex_follow_header,
+          severity: :warning,
+          category: :indexability,
+          url: page.url,
+          message: "X-Robots-Tag header prevents indexing but allows following links",
+          details: {content: content}
+        )
+      end
+
+      def report_noindex_nofollow_header(page, issues, content)
+        issues.add(
+          code: :noindex_nofollow_header,
+          severity: :error,
+          category: :indexability,
+          url: page.url,
+          message: "X-Robots-Tag header prevents indexing and following links",
+          details: {content: content}
+        )
+      end
+
+      def report_sitemap_noindex_url(page, issues, content, source:, sitemap_urls:)
+        normalized_url = Url.normalize(page.url, base_url: page.url)
+        return unless sitemap_urls.include?(normalized_url)
+
+        issues.add(
+          code: :sitemap_noindex_url,
+          severity: :error,
+          category: :sitemaps,
+          url: page.url,
+          message: "sitemap URL is noindex",
+          details: {content: content, source: source}
+        )
       end
     end
   end