crawlscope 0.5.0 → 0.7.0

data/test/crawlscope/reporter_test.rb

@@ -23,11 +23,25 @@ class CrawlscopeReporterTest < Minitest::Test
     refute_includes output, "Status: FAILED"
   end
 
-  def test_reports_failed_result_with_grouped_counts_and_offenses
+  def test_reports_warning_result_with_grouped_one_line_issues
     io = StringIO.new
     issues = Crawlscope::IssueCollection.new
+    4.times do |index|
+      issues.add(
+        code: :low_dofollow_inlinks,
+        severity: :warning,
+        category: :links,
+        url: "https://example.com/page-#{index + 1}",
+        message: "dofollow inbound links 1 below 2",
+        details: {
+          dofollow_inbound_count: 1,
+          minimum: 2,
+          source_urls: ["https://example.com/source-#{index + 1}"]
+        }
+      )
+    end
     issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com/a", message: "missing <title>", details: {})
-    issues.add(code: :broken_internal_link, severity: :notice, category: :links, url: "https://example.com/b", message: "broken internal link", details: {})
+
     result = Crawlscope::Result.new(
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
@@ -40,15 +54,126 @@ class CrawlscopeReporterTest < Minitest::Test
 
     output = io.string
 
+    assert_includes output, "Status: WARNINGS"
+    refute_includes output, "Status: FAILED"
+    assert_includes output, "Issues: 5 total (5 warnings)"
+    assert_includes output, "Summary:"
+    assert_includes output, "links / low_dofollow_inlinks: 4"
+    assert_includes output, "  - /page-1  inbound 1/2  sources: /source-1"
+    assert_includes output, "  - /page-4  inbound 1/2  sources: /source-4"
+    assert_includes output, "metadata / missing_title: 1"
+    refute_includes output, "Severity:"
+    refute_includes output, "Category:"
+    refute_includes output, "... 1 more"
+  end
+
+  def test_reports_failed_status_when_errors_are_present
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    issues.add(code: :fetch_failed, severity: :error, category: :crawl, url: "https://example.com/a", message: "timeout", details: {})
+    issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com/a", message: "missing <title>", details: {})
+
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com/a"],
+      pages: [Object.new],
+      issues: issues
+    )
+
+    Crawlscope::Reporter.new(io: io).report(result)
+
+    output = io.string
+
     assert_includes output, "Status: FAILED"
-    assert_includes output, "Issues: 2"
-    assert_includes output, "Severity:"
-    assert_includes output, "notice: 1"
-    assert_includes output, "warning: 1"
-    assert_includes output, "Category:"
-    assert_includes output, "links: 1"
-    assert_includes output, "metadata: 1"
-    assert_includes output, "  - [warning] missing_title https://example.com/a missing <title>"
-    assert_includes output, "  - [notice] broken_internal_link https://example.com/b broken internal link"
+    assert_includes output, "Issues: 2 total (1 error, 1 warning)"
+  end
+
+  def test_limits_large_issue_groups
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    21.times do |index|
+      issues.add(
+        code: :low_dofollow_inlinks,
+        severity: :warning,
+        category: :links,
+        url: "https://example.com/page-#{index + 1}",
+        message: "dofollow inbound links 1 below 2",
+        details: {dofollow_inbound_count: 1, minimum: 2}
+      )
+    end
+
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com"],
+      pages: [Object.new],
+      issues: issues
+    )
+
+    Crawlscope::Reporter.new(io: io).report(result)
+
+    output = io.string
+
+    assert_includes output, "links / low_dofollow_inlinks: 21"
+    assert_includes output, "  - /page-20  inbound 1/2"
+    refute_includes output, "  - /page-21"
+    assert_includes output, "  ... 1 more"
+  end
+
+  def test_reports_ratio_with_enough_precision_to_show_threshold_difference
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    issues.add(
+      code: :low_unique_token_ratio,
+      severity: :warning,
+      category: :content_quality,
+      url: "https://example.com/a",
+      message: "visible text has low token variety",
+      details: {ratio: 0.249, threshold: 0.25}
+    )
+
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com/a"],
+      pages: [Object.new],
+      issues: issues
+    )
+
+    Crawlscope::Reporter.new(io: io).report(result)
+
+    assert_includes io.string, "ratio 0.249/0.250"
+  end
+
+  def test_reports_source_details_on_one_line
+    io = StringIO.new
+    issues = Crawlscope::IssueCollection.new
+    4.times do |index|
+      issues.add(
+        code: :indexable_page_missing_from_sitemap,
+        severity: :warning,
+        category: :sitemaps,
+        url: "https://example.com/overview-#{index + 1}",
+        message: "indexable internal page is missing from sitemap",
+        details: {source_url: "https://example.com/source-#{index + 1}"}
+      )
+    end
+
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com"],
+      pages: [Object.new],
+      issues: issues
+    )
+
+    Crawlscope::Reporter.new(io: io).report(result)
+
+    output = io.string
+
+    assert_includes output, "sitemaps / indexable_page_missing_from_sitemap: 4"
+    assert_includes output, "  - /overview-1  indexable internal page is missing from sitemap  source: /source-1"
+    assert_includes output, "  - /overview-4  indexable internal page is missing from sitemap  source: /source-4"
   end
 end

data/test/crawlscope/result_test.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeResultTest < Minitest::Test
+  def test_ok_when_result_has_warnings_only
+    issues = Crawlscope::IssueCollection.new
+    issues.add(code: :missing_title, severity: :warning, category: :metadata, url: "https://example.com", message: "missing <title>", details: {})
+
+    result = result_with(issues)
+
+    assert result.ok?
+  end
+
+  def test_not_ok_when_result_has_errors
+    issues = Crawlscope::IssueCollection.new
+    issues.add(code: :fetch_failed, severity: :error, category: :crawl, url: "https://example.com", message: "timeout", details: {})
+
+    result = result_with(issues)
+
+    refute result.ok?
+  end
+
+  private
+
+  def result_with(issues)
+    Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: ["https://example.com"],
+      pages: [Object.new],
+      issues: issues
+    )
+  end
+end

data/test/crawlscope/sitemap_test.rb

@@ -3,6 +3,19 @@
 require "test_helper"
 
 class CrawlscopeSitemapTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :batches
+
+    def initialize
+      @batches = []
+    end
+
+    def call(items)
+      @batches << items
+      items.map { |item| yield(item) }
+    end
+  end
+
   def test_parses_remote_sitemap_urlset
     stub_request(:get, "https://www.example.com/sitemap.xml")
       .to_return(
@@ -127,4 +140,43 @@ class CrawlscopeSitemapTest < Minitest::Test
       assert_equal ["http://localhost:3000/features/reviews"], parser.urls(base_url: "http://localhost:3000")
     end
   end
+
+  def test_child_sitemaps_are_collected_through_the_fetch_executor
+    Dir.mktmpdir do |dir|
+      File.write(
+        File.join(dir, "sitemap.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>first.xml</loc></sitemap>
+            <sitemap><loc>second.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+      File.write(
+        File.join(dir, "first.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>http://localhost:3000/first</loc></url>
+          </urlset>
+        XML
+      )
+      File.write(
+        File.join(dir, "second.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>http://localhost:3000/second</loc></url>
+          </urlset>
+        XML
+      )
+
+      executor = RecordingExecutor.new
+      parser = Crawlscope::Sitemap.new(path: File.join(dir, "sitemap.xml"), fetch_executor: executor)
+
+      assert_equal ["http://localhost:3000/first", "http://localhost:3000/second"], parser.urls(base_url: "http://localhost:3000")
+      assert_equal [[File.join(dir, "first.xml"), File.join(dir, "second.xml")]], executor.batches
+    end
+  end
 end

data/test/performance/async_fetch_benchmark.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+require "time"
+
+class DelayedHttpServer
+  attr_reader :base_url
+
+  def initialize(page_count:, delay_seconds:)
+    @page_count = page_count
+    @delay_seconds = delay_seconds
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+
+  private
+
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_xml, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, page_html(path), content_type: "text/html")
+    end
+  ensure
+    socket.close
+  end
+
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+
+  def sitemap_xml
+    urls = (1..@page_count).map do |index|
+      "<url><loc>#{@base_url}/pages/#{index}</loc></url>"
+    end.join
+
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{urls}</urlset>)
+  end
+
+  def page_html(path)
+    <<~HTML
+      <html>
+        <head><title>#{path}</title></head>
+        <body><main><h1>#{path}</h1><p>#{path} benchmark page</p></main></body>
+      </html>
+    HTML
+  end
+
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+
+def measure(name, base_url:, concurrency:, fetch_executor:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+  result = Crawlscope::Crawl.new(
+    base_url: base_url,
+    sitemap_path: "#{base_url}/sitemap.xml",
+    rules: [],
+    schema_registry: Crawlscope::SchemaRegistry.default,
+    concurrency: concurrency,
+    fetch_executor: fetch_executor
+  ).call
+
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  [name, {seconds: elapsed.round(3), pages: result.pages.size, issues: result.issues.size}]
+end
+
+server = DelayedHttpServer.new(page_count: 24, delay_seconds: 0.08)
+server.start
+
+begin
+  results = {}
+  [
+    measure("threaded_concurrency_1", base_url: server.base_url, concurrency: 1, fetch_executor: :threaded),
+    measure("threaded_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :threaded),
+    measure("async_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :async)
+  ].each { |name, result| results[name] = result }
+
+  sequential = results.fetch("threaded_concurrency_1").fetch(:seconds)
+  threaded = results.fetch("threaded_concurrency_8").fetch(:seconds)
+  async = results.fetch("async_concurrency_8").fetch(:seconds)
+
+  abort "async benchmark failed: async was not meaningfully faster than sequential" unless async < sequential * 0.6
+  abort "async benchmark failed: async was more than 2x slower than threaded" if async > threaded * 2.0
+
+  puts JSON.pretty_generate(results)
+ensure
+  server.stop
+end

data/test/performance/fetch_executor_matrix.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+
+class MatrixHttpServer
+  attr_reader :base_url
+
+  def initialize(page_count:, delay_seconds:, link_targets: false)
+    @page_count = page_count
+    @delay_seconds = delay_seconds
+    @link_targets = link_targets
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+
+  private
+
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_xml, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, page_html(path), content_type: "text/html")
+    end
+  ensure
+    socket.close
+  end
+
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+
+  def sitemap_xml
+    paths = @link_targets ? ["/seed"] : (1..@page_count).map { |index| "/pages/#{index}" }
+    urls = paths.map { |path| "<url><loc>#{@base_url}#{path}</loc></url>" }.join
+
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{urls}</urlset>)
+  end
+
+  def page_html(path)
+    links = if @link_targets && path == "/seed"
+      (1..@page_count).map { |index| %(<a href="/targets/#{index}">Target #{index}</a>) }.join
+    else
+      ""
+    end
+
+    <<~HTML
+      <html>
+        <head>
+          <title>#{path}</title>
+          <meta name="robots" content="noindex">
+        </head>
+        <body>
+          <main><h1>#{path}</h1><p>#{path} benchmark page</p>#{links}</main>
+        </body>
+      </html>
+    HTML
+  end
+
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+
+def measure(base_url:, concurrency:, fetch_executor:, rules:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+  result = Crawlscope::Crawl.new(
+    base_url: base_url,
+    sitemap_path: "#{base_url}/sitemap.xml",
+    rules: rules,
+    schema_registry: Crawlscope::SchemaRegistry.default,
+    concurrency: concurrency,
+    fetch_executor: fetch_executor
+  ).call
+
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  {seconds: elapsed, pages: result.pages.size, issues: result.issues.size}
+end
+
+def median(values)
+  sorted = values.sort
+  sorted[sorted.length / 2]
+end
+
+def run_case(name:, page_count:, delay_seconds:, concurrency:, link_targets: false)
+  server = MatrixHttpServer.new(page_count: page_count, delay_seconds: delay_seconds, link_targets: link_targets)
+  server.start
+
+  rules = link_targets ? [Crawlscope::Rules::Links.new] : []
+  threaded = []
+  async = []
+
+  3.times do
+    threaded << measure(base_url: server.base_url, concurrency: concurrency, fetch_executor: :threaded, rules: rules)
+    async << measure(base_url: server.base_url, concurrency: concurrency, fetch_executor: :async, rules: rules)
+  end
+
+  threaded_seconds = median(threaded.map { |result| result.fetch(:seconds) })
+  async_seconds = median(async.map { |result| result.fetch(:seconds) })
+
+  {
+    name: name,
+    page_count: page_count,
+    delay_seconds: delay_seconds,
+    concurrency: concurrency,
+    link_targets: link_targets,
+    threaded_seconds: threaded_seconds.round(3),
+    async_seconds: async_seconds.round(3),
+    async_vs_threaded: (threaded_seconds / async_seconds).round(2),
+    pages: async.first.fetch(:pages),
+    issues: async.first.fetch(:issues)
+  }
+ensure
+  server&.stop
+end
+
+cases = [
+  {name: "direct_pages_c8", page_count: 48, delay_seconds: 0.02, concurrency: 8},
+  {name: "direct_pages_c16", page_count: 48, delay_seconds: 0.02, concurrency: 16},
+  {name: "slow_direct_pages_c8", page_count: 48, delay_seconds: 0.08, concurrency: 8},
+  {name: "slow_direct_pages_c16", page_count: 48, delay_seconds: 0.08, concurrency: 16},
+  {name: "link_targets_c8", page_count: 48, delay_seconds: 0.02, concurrency: 8, link_targets: true},
+  {name: "slow_link_targets_c8", page_count: 48, delay_seconds: 0.08, concurrency: 8, link_targets: true}
+]
+
+puts JSON.pretty_generate(cases.map { |attributes| run_case(**attributes) })

data/test/performance/sitemap_expansion_benchmark.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+
+class DelayedSitemapServer
+  attr_reader :base_url
+
+  def initialize(child_count:, delay_seconds:)
+    @child_count = child_count
+    @delay_seconds = delay_seconds
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+
+  private
+
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_index, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, child_sitemap(path), content_type: "application/xml")
+    end
+  ensure
+    socket.close
+  end
+
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+
+  def sitemap_index
+    children = (1..@child_count).map do |index|
+      "<sitemap><loc>#{@base_url}/sitemaps/#{index}.xml</loc></sitemap>"
+    end.join
+
+    %(<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{children}</sitemapindex>)
+  end
+
+  def child_sitemap(path)
+    index = File.basename(path, ".xml")
+
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>#{@base_url}/pages/#{index}</loc></url></urlset>)
+  end
+
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+
+def measure(name, base_url:, concurrency:, fetch_executor:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+  urls = Crawlscope::Sitemap.new(
+    path: "#{base_url}/sitemap.xml",
+    concurrency: concurrency,
+    fetch_executor: fetch_executor,
+    timeout_seconds: 5
+  ).urls(base_url: base_url)
+
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  [name, {seconds: elapsed.round(3), urls: urls.size}]
+end
+
+server = DelayedSitemapServer.new(child_count: 24, delay_seconds: 0.08)
+server.start
+
+begin
+  results = {}
+  [
+    measure("threaded_concurrency_1", base_url: server.base_url, concurrency: 1, fetch_executor: :threaded),
+    measure("threaded_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :threaded),
+    measure("async_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :async)
+  ].each { |name, result| results[name] = result }
+
+  sequential = results.fetch("threaded_concurrency_1").fetch(:seconds)
+  threaded = results.fetch("threaded_concurrency_8").fetch(:seconds)
+  async = results.fetch("async_concurrency_8").fetch(:seconds)
+
+  abort "sitemap benchmark failed: threaded parallelism was not at least 2x faster" unless threaded < sequential * 0.5
+  abort "sitemap benchmark failed: async parallelism was not at least 2x faster" unless async < sequential * 0.5
+
+  puts JSON.pretty_generate(results)
+ensure
+  server.stop
+end