RubyGems - crawlscope - Versions diffs - 0.4.0 → 0.6.0 - Mend

crawlscope 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +36 -0
data/README.md +38 -0
data/lib/crawlscope/cli.rb +20 -1
data/lib/crawlscope/configuration.rb +10 -1
data/lib/crawlscope/context.rb +1 -1
data/lib/crawlscope/crawl.rb +74 -14
data/lib/crawlscope/crawler.rb +3 -17
data/lib/crawlscope/document_text.rb +7 -2
data/lib/crawlscope/fetch_executor/async.rb +32 -0
data/lib/crawlscope/fetch_executor/threaded.rb +32 -0
data/lib/crawlscope/fetch_executor.rb +43 -0
data/lib/crawlscope/http.rb +7 -1
data/lib/crawlscope/rake_tasks.rb +27 -12
data/lib/crawlscope/reporter.rb +131 -7
data/lib/crawlscope/result.rb +1 -1
data/lib/crawlscope/rules/content_quality.rb +1 -1
data/lib/crawlscope/rules/indexability.rb +155 -20
data/lib/crawlscope/rules/links.rb +379 -12
data/lib/crawlscope/rules/metadata.rb +61 -6
data/lib/crawlscope/rules/structured_data.rb +31 -0
data/lib/crawlscope/rules/uniqueness.rb +45 -4
data/lib/crawlscope/sitemap.rb +39 -12
data/lib/crawlscope/version.rb +1 -1
data/lib/tasks/crawlscope_tasks.rake +24 -24
data/test/crawlscope/cli_test.rb +29 -2
data/test/crawlscope/configuration_test.rb +21 -0
data/test/crawlscope/content_quality_rule_test.rb +18 -0
data/test/crawlscope/crawl_test.rb +167 -3
data/test/crawlscope/crawler_test.rb +61 -0
data/test/crawlscope/fetch_executor_test.rb +44 -0
data/test/crawlscope/indexability_rule_test.rb +33 -0
data/test/crawlscope/links_rule_test.rb +249 -3
data/test/crawlscope/metadata_rule_test.rb +36 -0
data/test/crawlscope/rake_tasks_test.rb +70 -0
data/test/crawlscope/reporter_test.rb +136 -7
data/test/crawlscope/result_test.rb +35 -0
data/test/crawlscope/sitemap_test.rb +76 -0
data/test/crawlscope/structured_data_rule_test.rb +56 -0
data/test/crawlscope/uniqueness_rule_test.rb +17 -2
data/test/performance/async_fetch_benchmark.rb +127 -0
data/test/performance/fetch_executor_matrix.rb +162 -0
data/test/performance/sitemap_expansion_benchmark.rb +121 -0
metadata +39 -2

data/test/crawlscope/fetch_executor_test.rb ADDED Viewed

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+require "test_helper"
+class CrawlscopeFetchExecutorTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :items
+    def call(items)
+      @items = items
+      items.map { |item| yield(item) }
+    end
+  end
+  def test_map_preserves_input_order
+    results = Crawlscope::FetchExecutor.map(name: :threaded, concurrency: 2, items: [3, 1, 2]) do |item|
+      item * 10
+    end
+    assert_equal [30, 10, 20], results
+  end
+  def test_map_uses_sequential_fallback_for_single_item
+    executor = RecordingExecutor.new
+    results = Crawlscope::FetchExecutor.map(name: executor, concurrency: 4, items: ["one"]) do |item|
+      item.upcase
+    end
+    assert_equal ["ONE"], results
+    assert_nil executor.items
+  end
+  def test_map_uses_injected_executor_for_parallel_work
+    executor = RecordingExecutor.new
+    results = Crawlscope::FetchExecutor.map(name: executor, concurrency: 4, items: %w[a b]) do |item|
+      item.upcase
+    end
+    assert_equal %w[A B], results
+    assert_equal %w[a b], executor.items
+  end
+end

data/test/crawlscope/indexability_rule_test.rb CHANGED Viewed

@@ -20,6 +20,39 @@ class CrawlscopeIndexabilityRuleTest < Minitest::Test
     assert_equal :noindex_meta, issue.code
     assert_equal :error, issue.severity
     assert_equal "noindex, follow", issue.details[:content]
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :noindex_follow_meta
+    assert_includes codes, :sitemap_noindex_url
+  end
+  def test_reports_meta_nofollow
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      body: <<~HTML
+        <html>
+          <head><meta name="robots" content="nofollow"></head>
+          <body><main>Visible content</main></body>
+        </html>
+      HTML
+    )
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+    assert_equal [:nofollow_meta], issues.to_a.map(&:code)
+  end
+  def test_reports_noindex_nofollow_header
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(headers: {"X-Robots-Tag" => "googlebot: noindex, nofollow"})
+    Crawlscope::Rules::Indexability.new.call(urls: [page.url], pages: [page], issues: issues)
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :noindex_header
+    assert_includes codes, :nofollow_header
+    assert_includes codes, :noindex_nofollow_header
+    assert_includes codes, :sitemap_noindex_url
   end
   def test_reports_x_robots_tag_noindex

data/test/crawlscope/links_rule_test.rb CHANGED Viewed

@@ -41,7 +41,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
       context: context
     )
-    assert_equal [:broken_internal_link], issues.to_a.map(&:code)
+    assert_includes issues.to_a.map(&:code), :broken_internal_link
     assert_includes issues.to_a.first.message, "HTTP 404"
   end
@@ -114,8 +114,226 @@ class CrawlscopeLinksRuleTest < Minitest::Test
       context: context
     )
-    assert_equal [:low_inbound_anchor_links], issues.to_a.map(&:code)
-    assert_equal "https://example.com/guide", issues.to_a.first.url
+    orphan_issue = issues.to_a.find { |item| item.code == :orphan_page }
+    assert orphan_issue
+    assert_includes issues.to_a.map(&:code), :low_dofollow_inlinks
+    assert_equal "https://example.com/guide", orphan_issue.url
+  end
+  def test_reports_pages_with_no_outgoing_internal_links
+    issues = Crawlscope::IssueCollection.new
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context
+    )
+    issue = issues.to_a.find { |item| item.code == :page_has_no_outgoing_links }
+    assert issue
+    assert_equal "https://example.com/pricing", issue.url
+  end
+  def test_reports_nofollow_outlinks_and_inlink_follow_mix
+    issues = Crawlscope::IssueCollection.new
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing", "https://example.com/about"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\" rel=\"nofollow\">Pricing</a><a href=\"/about\">About</a></main>"),
+        page(url: "https://example.com/about", body: "<main><a href=\"/pricing\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :nofollow_internal_outlinks
+    assert_includes codes, :mixed_follow_internal_inlinks
+  end
+  def test_reports_only_nofollow_internal_inlinks
+    issues = Crawlscope::IssueCollection.new
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/pricing"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/pricing\" rel=\"nofollow\">Pricing</a></main>"),
+        page(url: "https://example.com/pricing", body: "<main><p>Pricing</p></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+    assert_includes issues.to_a.map(&:code), :only_nofollow_internal_inlinks
+  end
+  def test_reports_https_pages_linking_to_internal_http_urls
+    issues = Crawlscope::IssueCollection.new
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"http://example.com/pricing\">Pricing</a></main>")],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: true, error: nil, final_url: target_url, status: 200} })
+    )
+    assert_includes issues.to_a.map(&:code), :http_internal_link
+  end
+  def test_reports_canonical_target_link_issues
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      redirects = target_url == "https://example.com/canonical-about"
+      status = redirects ? 301 : 200
+      final_url = redirects ? "https://example.com/about" : target_url
+      {crawled: false, error: nil, final_url: final_url, status: status}
+    end
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide", "https://example.com/about"],
+      pages: [
+        page(url: "https://example.com/guide", body: "<main><a href=\"/about\">About</a></main>"),
+        page(
+          url: "https://example.com/about",
+          body: <<~HTML
+            <html>
+              <head><link rel="canonical" href="https://example.com/canonical-about"></head>
+              <body><main><p>About</p></main></body>
+            </html>
+          HTML
+        )
+      ],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :canonical_no_internal_inlinks
+    assert_includes codes, :canonical_points_to_redirect
+  end
+  def test_does_not_report_missing_inlinks_for_root_canonical
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {crawled: true, error: nil, final_url: target_url, html: true, status: 200}
+    end
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/", "https://example.com/about"],
+      pages: [
+        page(
+          url: "https://example.com/",
+          body: <<~HTML
+            <html>
+              <head><link rel="canonical" href="https://example.com/"></head>
+              <body><main><a href="/about">About</a></main></body>
+            </html>
+          HTML
+        ),
+        page(url: "https://example.com/about", body: "<main><a href=\"/\">Home</a></main>")
+      ],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+    refute_includes issues.to_a.map(&:code), :canonical_no_internal_inlinks
+  end
+  def test_reports_indexable_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        error: nil,
+        final_url: target_url,
+        html: true,
+        status: 200
+      }
+    end
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+    issue = issues.to_a.find { |item| item.code == :indexable_page_missing_from_sitemap }
+    assert issue
+    assert_equal "https://example.com/hidden", issue.url
+  end
+  def test_does_not_report_noindex_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        doc: Nokogiri::HTML("<head><meta name=\"robots\" content=\"noindex, follow\"></head>"),
+        error: nil,
+        final_url: target_url,
+        headers: {},
+        html: true,
+        status: 200
+      }
+    end
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+    refute_includes issues.to_a.map(&:code), :indexable_page_missing_from_sitemap
+  end
+  def test_does_not_report_x_robots_noindex_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        doc: Nokogiri::HTML("<main>Hidden</main>"),
+        error: nil,
+        final_url: target_url,
+        headers: {"X-Robots-Tag" => "noindex"},
+        html: true,
+        status: 200
+      }
+    end
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+    refute_includes issues.to_a.map(&:code), :indexable_page_missing_from_sitemap
+  end
+  def test_reports_url_hygiene_issues
+    issues = Crawlscope::IssueCollection.new
+    long_path = "a" * 2_050
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com//bad", "https://example.com/#{long_path}"],
+      pages: [
+        page(url: "https://example.com//bad", body: "<main><a href=\"/ok\">OK</a></main>"),
+        page(url: "https://example.com/#{long_path}", body: "<main><a href=\"/ok\">OK</a></main>")
+      ],
+      issues: issues,
+      context: context(resolver: ->(target_url) { {crawled: false, error: nil, final_url: target_url, html: true, status: 200} })
+    )
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :url_double_slash
+    assert_includes codes, :url_too_long
   end
   def test_counts_root_page_links_as_inbound_links
@@ -157,6 +375,32 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     assert_includes redirect_issue.message, "https://example.com/pricing"
   end
+  def test_reuses_link_target_resolution_for_later_link_checks
+    issues = Crawlscope::IssueCollection.new
+    resolution_counts = Hash.new(0)
+    resolver = lambda do |target_url|
+      resolution_counts[target_url] += 1
+      {
+        crawled: false,
+        doc: Nokogiri::HTML("<main>Hidden</main>"),
+        error: nil,
+        final_url: target_url,
+        headers: {},
+        html: true,
+        status: 200
+      }
+    end
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+    assert_equal 1, resolution_counts.fetch("https://example.com/hidden")
+  end
   def test_ignores_links_that_should_not_be_crawled
     issues = Crawlscope::IssueCollection.new
@@ -217,6 +461,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
         crawled: true,
         error: nil,
         final_url: target_url,
+        html: true,
         status: 200
       }
     when "https://example.com/missing"
@@ -224,6 +469,7 @@ class CrawlscopeLinksRuleTest < Minitest::Test
         crawled: false,
         error: nil,
         final_url: target_url,
+        html: false,
         status: 404
       }
     end

data/test/crawlscope/metadata_rule_test.rb CHANGED Viewed

@@ -48,6 +48,42 @@ class CrawlscopeMetadataRuleTest < Minitest::Test
     refute_includes issues.to_a.map(&:code), :canonical_mismatch
   end
+  def test_reports_multiple_title_multiple_descriptions_empty_h1_and_sitemap_canonical_mismatch
+    issues = Crawlscope::IssueCollection.new
+    invalid_page = page(
+      body: <<~HTML
+        <html>
+          <head>
+            <title>About</title>
+            <title>Duplicate About</title>
+            <meta name="description" content="A clear description that is long enough for search snippets, local validation checks, and realistic production metadata audits.">
+            <meta name="description" content="Duplicate description">
+            <link rel="canonical" href="https://example.com/canonical-about">
+            <meta property="og:title" content="About">
+            <meta property="og:description" content="About page">
+            <meta property="og:url" content="https://example.com/about">
+            <meta property="og:type" content="website">
+            <meta property="og:image" content="https://example.com/icon.png">
+          </head>
+          <body><main><h1> </h1></main></body>
+        </html>
+      HTML
+    )
+    Crawlscope::Rules::Metadata.new.call(
+      urls: [invalid_page.url],
+      pages: [invalid_page],
+      issues: issues
+    )
+    codes = issues.to_a.map(&:code)
+    assert_includes codes, :multiple_title_tags
+    assert_includes codes, :multiple_meta_descriptions
+    assert_includes codes, :empty_h1
+    assert_includes codes, :canonical_mismatch
+    assert_includes codes, :non_canonical_page_in_sitemap
+  end
   private
   def page(url: "https://example.com/about", body: nil)

data/test/crawlscope/rake_tasks_test.rb ADDED Viewed

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+require "test_helper"
+class CrawlscopeRakeTasksTest < Minitest::Test
+  def setup
+    @original_start = Crawlscope::Cli.method(:start)
+  end
+  def teardown
+    singleton_class = class << Crawlscope::Cli; self; end
+    original_start = @original_start
+    singleton_class.define_method(:start) do |*args, **kwargs|
+      original_start.call(*args, **kwargs)
+    end
+  end
+  def test_validate_passes_rake_arguments_to_cli
+    calls = capture_cli_calls
+    Crawlscope::RakeTasks.validate(
+      url: "http://localhost:3001",
+      sitemap_path: "http://localhost:3001/sitemap.xml",
+      rule_names: "metadata,links"
+    )
+    assert_equal(
+      ["validate", "--url", "http://localhost:3001", "--sitemap", "http://localhost:3001/sitemap.xml", "--rules", "metadata,links"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+  def test_validate_rule_passes_rule_and_rake_arguments_to_cli
+    calls = capture_cli_calls
+    Crawlscope::RakeTasks.validate_rule(
+      "metadata",
+      url: "http://localhost:3001",
+      sitemap_path: "http://localhost:3001/sitemap.xml"
+    )
+    assert_equal(
+      ["validate", "--url", "http://localhost:3001", "--sitemap", "http://localhost:3001/sitemap.xml", "--rules", "metadata"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+  def test_ldjson_passes_rake_url_argument_to_cli
+    calls = capture_cli_calls
+    Crawlscope::RakeTasks.ldjson(urls: "http://localhost:3001/article")
+    assert_equal(
+      ["ldjson", "--url", "http://localhost:3001/article"],
+      calls.fetch(0).fetch(:argv)
+    )
+  end
+  private
+  def capture_cli_calls
+    calls = []
+    singleton_class = class << Crawlscope::Cli; self; end
+    singleton_class.define_method(:start) do |argv, **kwargs|
+      calls << {argv: argv, kwargs: kwargs}
+      0
+    end
+    calls
+  end
+end

data/test/crawlscope/reporter_test.rb CHANGED Viewed

@@ -23,11 +23,25 @@ class CrawlscopeReporterTest < Minitest::Test
     refute_includes output, "Status: FAILED"
   end
-  def test_reports_failed_result_with_severity_counts
+  def test_reports_warning_result_with_grouped_one_line_issues
     io = StringIO.new
     issues = Crawlscope::IssueCollection.new
+    4.times do |index|
+      issues.add(
+        code: :low_dofollow_inlinks,
+        severity: :warning,
+        category: :links,
+        url: "https://example.com/page-#{index + 1}",
+        message: "dofollow inbound links 1 below 2",
+        details: {
+          dofollow_inbound_count: 1,
+          minimum: 2,
+          source_urls: ["https://example.com/source-#{index + 1}"]
+        }
+      )
+    end
     issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com/a", message: "missing <title>", details: {})
-    issues.add(code: :broken_internal_link, severity: :notice, category: :links, url: "https://example.com/b", message: "broken internal link", details: {})
     result = Crawlscope::Result.new(
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
@@ -40,11 +54,126 @@ class CrawlscopeReporterTest < Minitest::Test
     output = io.string
+    assert_includes output, "Status: WARNINGS"
+    refute_includes output, "Status: FAILED"
+    assert_includes output, "Issues: 5 total (5 warnings)"
+    assert_includes output, "Summary:"
+    assert_includes output, "links / low_dofollow_inlinks: 4"
+    assert_includes output, "  - /page-1  inbound 1/2  sources: /source-1"
+    assert_includes output, "  - /page-4  inbound 1/2  sources: /source-4"
+    assert_includes output, "metadata / missing_title: 1"
+    refute_includes output, "Severity:"
+    refute_includes output, "Category:"
+    refute_includes output, "... 1 more"
+  end
+  def test_reports_failed_status_when_errors_are_present
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    issues.add(code: :fetch_failed, severity: :error, category: :crawl, url: "https://example.com/a", message: "timeout", details: {})
+    issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com/a", message: "missing <title>", details: {})
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com/a"],
+      pages: [Object.new],
+      issues: issues
+    )
+    Crawlscope::Reporter.new(io: io).report(result)
+    output = io.string
     assert_includes output, "Status: FAILED"
-    assert_includes output, "Issues: 2"
-    assert_includes output, "notice: 1"
-    assert_includes output, "warning: 1"
-    assert_includes output, "- [warning] https://example.com/a missing <title>"
-    assert_includes output, "- [notice] https://example.com/b broken internal link"
+    assert_includes output, "Issues: 2 total (1 error, 1 warning)"
+  end
+  def test_limits_large_issue_groups
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    21.times do |index|
+      issues.add(
+        code: :low_dofollow_inlinks,
+        severity: :warning,
+        category: :links,
+        url: "https://example.com/page-#{index + 1}",
+        message: "dofollow inbound links 1 below 2",
+        details: {dofollow_inbound_count: 1, minimum: 2}
+      )
+    end
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com"],
+      pages: [Object.new],
+      issues: issues
+    )
+    Crawlscope::Reporter.new(io: io).report(result)
+    output = io.string
+    assert_includes output, "links / low_dofollow_inlinks: 21"
+    assert_includes output, "  - /page-20  inbound 1/2"
+    refute_includes output, "  - /page-21"
+    assert_includes output, "  ... 1 more"
+  end
+  def test_reports_ratio_with_enough_precision_to_show_threshold_difference
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    issues.add(
+      code: :low_unique_token_ratio,
+      severity: :warning,
+      category: :content_quality,
+      url: "https://example.com/a",
+      message: "visible text has low token variety",
+      details: {ratio: 0.249, threshold: 0.25}
+    )
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com/a"],
+      pages: [Object.new],
+      issues: issues
+    )
+    Crawlscope::Reporter.new(io: io).report(result)
+    assert_includes io.string, "ratio 0.249/0.250"
+  end
+  def test_reports_source_details_on_one_line
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    4.times do |index|
+      issues.add(
+        code: :indexable_page_missing_from_sitemap,
+        severity: :warning,
+        category: :sitemaps,
+        url: "https://example.com/overview-#{index + 1}",
+        message: "indexable internal page is missing from sitemap",
+        details: {source_url: "https://example.com/source-#{index + 1}"}
+      )
+    end
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com"],
+      pages: [Object.new],
+      issues: issues
+    )
+    Crawlscope::Reporter.new(io: io).report(result)
+    output = io.string
+    assert_includes output, "sitemaps / indexable_page_missing_from_sitemap: 4"
+    assert_includes output, "  - /overview-1  indexable internal page is missing from sitemap  source: /source-1"
+    assert_includes output, "  - /overview-4  indexable internal page is missing from sitemap  source: /source-4"
   end
 end

data/test/crawlscope/result_test.rb ADDED Viewed

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+require "test_helper"
+class CrawlscopeResultTest < Minitest::Test
+  def test_ok_when_result_has_warnings_only
+    issues = Crawlscope::IssueCollection.new
+    issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com", message: "missing <title>", details: {})
+    result = result_with(issues)
+    assert result.ok?
+  end
+  def test_not_ok_when_result_has_errors
+    issues = Crawlscope::IssueCollection.new
+    issues.add(code: :fetch_failed, severity: :error, category: :crawl, url: "https://example.com", message: "timeout", details: {})
+    result = result_with(issues)
+    refute result.ok?
+  end
+  private
+  def result_with(issues)
+    Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com"],
+      pages: [Object.new],
+      issues: issues
+    )
+  end
+end