RubyGems - crawlscope - Versions diffs - 0.4.0 → 0.6.0 - Mend

crawlscope 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +36 -0
data/README.md +38 -0
data/lib/crawlscope/cli.rb +20 -1
data/lib/crawlscope/configuration.rb +10 -1
data/lib/crawlscope/context.rb +1 -1
data/lib/crawlscope/crawl.rb +74 -14
data/lib/crawlscope/crawler.rb +3 -17
data/lib/crawlscope/document_text.rb +7 -2
data/lib/crawlscope/fetch_executor/async.rb +32 -0
data/lib/crawlscope/fetch_executor/threaded.rb +32 -0
data/lib/crawlscope/fetch_executor.rb +43 -0
data/lib/crawlscope/http.rb +7 -1
data/lib/crawlscope/rake_tasks.rb +27 -12
data/lib/crawlscope/reporter.rb +131 -7
data/lib/crawlscope/result.rb +1 -1
data/lib/crawlscope/rules/content_quality.rb +1 -1
data/lib/crawlscope/rules/indexability.rb +155 -20
data/lib/crawlscope/rules/links.rb +379 -12
data/lib/crawlscope/rules/metadata.rb +61 -6
data/lib/crawlscope/rules/structured_data.rb +31 -0
data/lib/crawlscope/rules/uniqueness.rb +45 -4
data/lib/crawlscope/sitemap.rb +39 -12
data/lib/crawlscope/version.rb +1 -1
data/lib/tasks/crawlscope_tasks.rake +24 -24
data/test/crawlscope/cli_test.rb +29 -2
data/test/crawlscope/configuration_test.rb +21 -0
data/test/crawlscope/content_quality_rule_test.rb +18 -0
data/test/crawlscope/crawl_test.rb +167 -3
data/test/crawlscope/crawler_test.rb +61 -0
data/test/crawlscope/fetch_executor_test.rb +44 -0
data/test/crawlscope/indexability_rule_test.rb +33 -0
data/test/crawlscope/links_rule_test.rb +249 -3
data/test/crawlscope/metadata_rule_test.rb +36 -0
data/test/crawlscope/rake_tasks_test.rb +70 -0
data/test/crawlscope/reporter_test.rb +136 -7
data/test/crawlscope/result_test.rb +35 -0
data/test/crawlscope/sitemap_test.rb +76 -0
data/test/crawlscope/structured_data_rule_test.rb +56 -0
data/test/crawlscope/uniqueness_rule_test.rb +17 -2
data/test/performance/async_fetch_benchmark.rb +127 -0
data/test/performance/fetch_executor_matrix.rb +162 -0
data/test/performance/sitemap_expansion_benchmark.rb +121 -0
metadata +39 -2

data/test/crawlscope/sitemap_test.rb CHANGED Viewed

@@ -3,6 +3,19 @@
 require "test_helper"
 class CrawlscopeSitemapTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :batches
+    def initialize
+      @batches = []
+    end
+    def call(items)
+      @batches << items
+      items.map { |item| yield(item) }
+    end
+  end
   def test_parses_remote_sitemap_urlset
     stub_request(:get, "https://www.example.com/sitemap.xml")
       .to_return(
@@ -49,6 +62,30 @@ class CrawlscopeSitemapTest < Minitest::Test
     assert_equal ["https://www.example.com/features/reviews"], parser.urls(base_url: "https://www.example.com")
   end
+  def test_remote_sitemap_http_error_is_explicit
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(status: 500, body: "<html><body>Error</body></html>")
+    parser = Crawlscope::Sitemap.new(path: "https://www.example.com/sitemap.xml")
+    error = assert_raises(Crawlscope::ValidationError) do
+      parser.urls(base_url: "https://www.example.com")
+    end
+    assert_equal "Sitemap https://www.example.com/sitemap.xml returned HTTP 500", error.message
+  end
+  def test_invalid_sitemap_root_is_explicit
+    stub_request(:get, "https://www.example.com/sitemap.xml")
+      .to_return(status: 200, body: "<html><body>Error</body></html>")
+    parser = Crawlscope::Sitemap.new(path: "https://www.example.com/sitemap.xml")
+    error = assert_raises(Crawlscope::ValidationError) do
+      parser.urls(base_url: "https://www.example.com")
+    end
+    assert_equal 'Sitemap https://www.example.com/sitemap.xml has unexpected root "html"', error.message
+  end
   def test_rebases_remote_sitemap_index_children_to_base_url
     stub_request(:get, "http://localhost:3000/sitemap.xml")
       .to_return(
@@ -103,4 +140,43 @@ class CrawlscopeSitemapTest < Minitest::Test
       assert_equal ["http://localhost:3000/features/reviews"], parser.urls(base_url: "http://localhost:3000")
     end
   end
+  def test_child_sitemaps_are_collected_through_the_fetch_executor
+    Dir.mktmpdir do |dir|
+      File.write(
+        File.join(dir, "sitemap.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>first.xml</loc></sitemap>
+            <sitemap><loc>second.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+      File.write(
+        File.join(dir, "first.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>http://localhost:3000/first</loc></url>
+          </urlset>
+        XML
+      )
+      File.write(
+        File.join(dir, "second.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>http://localhost:3000/second</loc></url>
+          </urlset>
+        XML
+      )
+      executor = RecordingExecutor.new
+      parser = Crawlscope::Sitemap.new(path: File.join(dir, "sitemap.xml"), fetch_executor: executor)
+      assert_equal ["http://localhost:3000/first", "http://localhost:3000/second"], parser.urls(base_url: "http://localhost:3000")
+      assert_equal [[File.join(dir, "first.xml"), File.join(dir, "second.xml")]], executor.batches
+    end
+  end
 end

data/test/crawlscope/structured_data_rule_test.rb CHANGED Viewed

@@ -79,6 +79,62 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
     assert_equal ["json-ld", "microdata"], issues.to_a.first.details[:expected_sources]
   end
+  def test_reports_structured_data_missing_type
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","headline":"Untyped article"}
+            </script>
+          </head>
+          <body><h1>Article</h1></body>
+        </html>
+      HTML
+    )
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+    assert_includes issues.to_a.map(&:code), :structured_data_missing_type
+  end
+  def test_reports_graph_entries_missing_type
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::StructuredData.new
+    page = page(
+      url: "https://example.com/articles/test",
+      body: <<~HTML
+        <html>
+          <head>
+            <script type="application/ld+json">
+              {"@context":"https://schema.org","@type":"WebPage","@graph":[{"name":"Untyped node"}]}
+            </script>
+          </head>
+          <body><h1>Article</h1></body>
+        </html>
+      HTML
+    )
+    rule.call(
+      urls: [page.url],
+      pages: [page],
+      issues: issues,
+      context: {schema_registry: Crawlscope::SchemaRegistry.default}
+    )
+    issue = issues.to_a.find { |item| item.code == :structured_data_missing_type }
+    assert issue
+    assert_equal ["$.@graph[0]"], issue.details[:paths]
+  end
   def test_validates_job_posting_markup
     issues = Crawlscope::IssueCollection.new
     rule = Crawlscope::Rules::StructuredData.new

data/test/crawlscope/uniqueness_rule_test.rb CHANGED Viewed

@@ -13,7 +13,20 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
-    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_title].sort, issues.to_a.map(&:code).sort
+    assert_equal %i[duplicate_content_fingerprint duplicate_meta_description duplicate_pages_without_canonical duplicate_title].sort, issues.to_a.map(&:code).sort
+  end
+  def test_allows_duplicate_pages_when_canonicals_are_present
+    issues = Crawlscope::IssueCollection.new
+    rule = Crawlscope::Rules::Uniqueness.new
+    pages = [
+      page(url: "https://example.com/a", canonical: "https://example.com/a"),
+      page(url: "https://example.com/b", canonical: "https://example.com/a")
+    ]
+    rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+    refute_includes issues.to_a.map(&:code), :duplicate_pages_without_canonical
   end
   def test_reports_near_duplicate_content
@@ -59,13 +72,15 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     TEXT
   end
-  def page(url:, content: nil)
+  def page(url:, content: nil, canonical: nil)
     repeated_text = content || ("Useful content " * 30).strip
+    canonical_tag = canonical ? %(<link rel="canonical" href="#{canonical}">) : ""
     body = <<~HTML
       <html>
         <head>
           <title>Example Title</title>
           <meta name="description" content="Example description">
+          #{canonical_tag}
         </head>
         <body>
           <main>#{repeated_text}</main>

data/test/performance/async_fetch_benchmark.rb ADDED Viewed

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+require "time"
+class DelayedHttpServer
+  attr_reader :base_url
+  def initialize(page_count:, delay_seconds:)
+    @page_count = page_count
+    @delay_seconds = delay_seconds
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+  private
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_xml, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, page_html(path), content_type: "text/html")
+    end
+  ensure
+    socket.close
+  end
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+  def sitemap_xml
+    urls = (1..@page_count).map do |index|
+      "<url><loc>#{@base_url}/pages/#{index}</loc></url>"
+    end.join
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{urls}</urlset>)
+  end
+  def page_html(path)
+    <<~HTML
+      <html>
+        <head><title>#{path}</title></head>
+        <body><main><h1>#{path}</h1><p>#{path} benchmark page</p></main></body>
+      </html>
+    HTML
+  end
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+def measure(name, base_url:, concurrency:, fetch_executor:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  result = Crawlscope::Crawl.new(
+    base_url: base_url,
+    sitemap_path: "#{base_url}/sitemap.xml",
+    rules: [],
+    schema_registry: Crawlscope::SchemaRegistry.default,
+    concurrency: concurrency,
+    fetch_executor: fetch_executor
+  ).call
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  [name, {seconds: elapsed.round(3), pages: result.pages.size, issues: result.issues.size}]
+end
+server = DelayedHttpServer.new(page_count: 24, delay_seconds: 0.08)
+server.start
+begin
+  results = {}
+  [
+    measure("threaded_concurrency_1", base_url: server.base_url, concurrency: 1, fetch_executor: :threaded),
+    measure("threaded_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :threaded),
+    measure("async_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :async)
+  ].each { |name, result| results[name] = result }
+  sequential = results.fetch("threaded_concurrency_1").fetch(:seconds)
+  threaded = results.fetch("threaded_concurrency_8").fetch(:seconds)
+  async = results.fetch("async_concurrency_8").fetch(:seconds)
+  abort "async benchmark failed: async was not meaningfully faster than sequential" unless async < sequential * 0.6
+  abort "async benchmark failed: async was more than 2x slower than threaded" if async > threaded * 2.0
+  puts JSON.pretty_generate(results)
+ensure
+  server.stop
+end

data/test/performance/fetch_executor_matrix.rb ADDED Viewed

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+class MatrixHttpServer
+  attr_reader :base_url
+  def initialize(page_count:, delay_seconds:, link_targets: false)
+    @page_count = page_count
+    @delay_seconds = delay_seconds
+    @link_targets = link_targets
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+  private
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_xml, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, page_html(path), content_type: "text/html")
+    end
+  ensure
+    socket.close
+  end
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+  def sitemap_xml
+    paths = @link_targets ? ["/seed"] : (1..@page_count).map { |index| "/pages/#{index}" }
+    urls = paths.map { |path| "<url><loc>#{@base_url}#{path}</loc></url>" }.join
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{urls}</urlset>)
+  end
+  def page_html(path)
+    links = if @link_targets && path == "/seed"
+      (1..@page_count).map { |index| %(<a href="/targets/#{index}">Target #{index}</a>) }.join
+    else
+      ""
+    end
+    <<~HTML
+      <html>
+        <head>
+          <title>#{path}</title>
+          <meta name="robots" content="noindex">
+        </head>
+        <body>
+          <main><h1>#{path}</h1><p>#{path} benchmark page</p>#{links}</main>
+        </body>
+      </html>
+    HTML
+  end
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+def measure(base_url:, concurrency:, fetch_executor:, rules:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  result = Crawlscope::Crawl.new(
+    base_url: base_url,
+    sitemap_path: "#{base_url}/sitemap.xml",
+    rules: rules,
+    schema_registry: Crawlscope::SchemaRegistry.default,
+    concurrency: concurrency,
+    fetch_executor: fetch_executor
+  ).call
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  {seconds: elapsed, pages: result.pages.size, issues: result.issues.size}
+end
+def median(values)
+  sorted = values.sort
+  sorted[sorted.length / 2]
+end
+def run_case(name:, page_count:, delay_seconds:, concurrency:, link_targets: false)
+  server = MatrixHttpServer.new(page_count: page_count, delay_seconds: delay_seconds, link_targets: link_targets)
+  server.start
+  rules = link_targets ? [Crawlscope::Rules::Links.new] : []
+  threaded = []
+  async = []
+  3.times do
+    threaded << measure(base_url: server.base_url, concurrency: concurrency, fetch_executor: :threaded, rules: rules)
+    async << measure(base_url: server.base_url, concurrency: concurrency, fetch_executor: :async, rules: rules)
+  end
+  threaded_seconds = median(threaded.map { |result| result.fetch(:seconds) })
+  async_seconds = median(async.map { |result| result.fetch(:seconds) })
+  {
+    name: name,
+    page_count: page_count,
+    delay_seconds: delay_seconds,
+    concurrency: concurrency,
+    link_targets: link_targets,
+    threaded_seconds: threaded_seconds.round(3),
+    async_seconds: async_seconds.round(3),
+    async_vs_threaded: (threaded_seconds / async_seconds).round(2),
+    pages: async.first.fetch(:pages),
+    issues: async.first.fetch(:issues)
+  }
+ensure
+  server&.stop
+end
+cases = [
+  {name: "direct_pages_c8", page_count: 48, delay_seconds: 0.02, concurrency: 8},
+  {name: "direct_pages_c16", page_count: 48, delay_seconds: 0.02, concurrency: 16},
+  {name: "slow_direct_pages_c8", page_count: 48, delay_seconds: 0.08, concurrency: 8},
+  {name: "slow_direct_pages_c16", page_count: 48, delay_seconds: 0.08, concurrency: 16},
+  {name: "link_targets_c8", page_count: 48, delay_seconds: 0.02, concurrency: 8, link_targets: true},
+  {name: "slow_link_targets_c8", page_count: 48, delay_seconds: 0.08, concurrency: 8, link_targets: true}
+]
+puts JSON.pretty_generate(cases.map { |attributes| run_case(**attributes) })

data/test/performance/sitemap_expansion_benchmark.rb ADDED Viewed

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+class DelayedSitemapServer
+  attr_reader :base_url
+  def initialize(child_count:, delay_seconds:)
+    @child_count = child_count
+    @delay_seconds = delay_seconds
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+  private
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_index, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, child_sitemap(path), content_type: "application/xml")
+    end
+  ensure
+    socket.close
+  end
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+  def sitemap_index
+    children = (1..@child_count).map do |index|
+      "<sitemap><loc>#{@base_url}/sitemaps/#{index}.xml</loc></sitemap>"
+    end.join
+    %(<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{children}</sitemapindex>)
+  end
+  def child_sitemap(path)
+    index = File.basename(path, ".xml")
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>#{@base_url}/pages/#{index}</loc></url></urlset>)
+  end
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+def measure(name, base_url:, concurrency:, fetch_executor:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  urls = Crawlscope::Sitemap.new(
+    path: "#{base_url}/sitemap.xml",
+    concurrency: concurrency,
+    fetch_executor: fetch_executor,
+    timeout_seconds: 5
+  ).urls(base_url: base_url)
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  [name, {seconds: elapsed.round(3), urls: urls.size}]
+end
+server = DelayedSitemapServer.new(child_count: 24, delay_seconds: 0.08)
+server.start
+begin
+  results = {}
+  [
+    measure("threaded_concurrency_1", base_url: server.base_url, concurrency: 1, fetch_executor: :threaded),
+    measure("threaded_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :threaded),
+    measure("async_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :async)
+  ].each { |name, result| results[name] = result }
+  sequential = results.fetch("threaded_concurrency_1").fetch(:seconds)
+  threaded = results.fetch("threaded_concurrency_8").fetch(:seconds)
+  async = results.fetch("async_concurrency_8").fetch(:seconds)
+  abort "sitemap benchmark failed: threaded parallelism was not at least 2x faster" unless threaded < sequential * 0.5
+  abort "sitemap benchmark failed: async parallelism was not at least 2x faster" unless async < sequential * 0.5
+  puts JSON.pretty_generate(results)
+ensure
+  server.stop
+end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -24,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: async-http-faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.22'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -200,6 +228,9 @@ files:
 - lib/crawlscope/crawl.rb
 - lib/crawlscope/crawler.rb
 - lib/crawlscope/document_text.rb
+- lib/crawlscope/fetch_executor.rb
+- lib/crawlscope/fetch_executor/async.rb
+- lib/crawlscope/fetch_executor/threaded.rb
 - lib/crawlscope/http.rb
 - lib/crawlscope/issue.rb
 - lib/crawlscope/issue_collection.rb
@@ -234,12 +265,15 @@ files:
 - test/crawlscope/content_quality_rule_test.rb
 - test/crawlscope/crawl_test.rb
 - test/crawlscope/crawler_test.rb
+- test/crawlscope/fetch_executor_test.rb
 - test/crawlscope/http_test.rb
 - test/crawlscope/indexability_rule_test.rb
 - test/crawlscope/links_rule_test.rb
 - test/crawlscope/loader_test.rb
 - test/crawlscope/metadata_rule_test.rb
+- test/crawlscope/rake_tasks_test.rb
 - test/crawlscope/reporter_test.rb
+- test/crawlscope/result_test.rb
 - test/crawlscope/rule_registry_test.rb
 - test/crawlscope/run_test.rb
 - test/crawlscope/schema_registry_test.rb
@@ -252,6 +286,9 @@ files:
 - test/crawlscope/structured_data_writer_test.rb
 - test/crawlscope/uniqueness_rule_test.rb
 - test/crawlscope/url_test.rb
+- test/performance/async_fetch_benchmark.rb
+- test/performance/fetch_executor_matrix.rb
+- test/performance/sitemap_expansion_benchmark.rb
 - test/release_task_test.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/crawlscope
@@ -274,7 +311,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="