RubyGems - crawlscope - Versions diffs - 0.5.0 → 0.6.0 - Mend

crawlscope 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (36) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +27 -0
data/README.md +32 -0
data/lib/crawlscope/cli.rb +16 -0
data/lib/crawlscope/configuration.rb +10 -1
data/lib/crawlscope/context.rb +1 -1
data/lib/crawlscope/crawl.rb +72 -14
data/lib/crawlscope/crawler.rb +3 -17
data/lib/crawlscope/document_text.rb +7 -2
data/lib/crawlscope/fetch_executor/async.rb +32 -0
data/lib/crawlscope/fetch_executor/threaded.rb +32 -0
data/lib/crawlscope/fetch_executor.rb +43 -0
data/lib/crawlscope/http.rb +7 -1
data/lib/crawlscope/reporter.rb +123 -14
data/lib/crawlscope/result.rb +1 -1
data/lib/crawlscope/rules/content_quality.rb +1 -1
data/lib/crawlscope/rules/indexability.rb +28 -6
data/lib/crawlscope/rules/links.rb +80 -16
data/lib/crawlscope/rules/uniqueness.rb +23 -4
data/lib/crawlscope/sitemap.rb +30 -11
data/lib/crawlscope/version.rb +1 -1
data/lib/tasks/crawlscope_tasks.rake +1 -1
data/test/crawlscope/cli_test.rb +28 -2
data/test/crawlscope/configuration_test.rb +21 -0
data/test/crawlscope/content_quality_rule_test.rb +18 -0
data/test/crawlscope/crawl_test.rb +142 -4
data/test/crawlscope/crawler_test.rb +61 -0
data/test/crawlscope/fetch_executor_test.rb +44 -0
data/test/crawlscope/links_rule_test.rb +101 -0
data/test/crawlscope/reporter_test.rb +136 -11
data/test/crawlscope/result_test.rb +35 -0
data/test/crawlscope/sitemap_test.rb +52 -0
data/test/performance/async_fetch_benchmark.rb +127 -0
data/test/performance/fetch_executor_matrix.rb +162 -0
data/test/performance/sitemap_expansion_benchmark.rb +121 -0
metadata +38 -2

data/test/crawlscope/sitemap_test.rb CHANGED Viewed

@@ -3,6 +3,19 @@
 require "test_helper"
 class CrawlscopeSitemapTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :batches
+    def initialize
+      @batches = []
+    end
+    def call(items)
+      @batches << items
+      items.map { |item| yield(item) }
+    end
+  end
   def test_parses_remote_sitemap_urlset
     stub_request(:get, "https://www.example.com/sitemap.xml")
       .to_return(
@@ -127,4 +140,43 @@ class CrawlscopeSitemapTest < Minitest::Test
       assert_equal ["http://localhost:3000/features/reviews"], parser.urls(base_url: "http://localhost:3000")
     end
   end
+  def test_child_sitemaps_are_collected_through_the_fetch_executor
+    Dir.mktmpdir do |dir|
+      File.write(
+        File.join(dir, "sitemap.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <sitemap><loc>first.xml</loc></sitemap>
+            <sitemap><loc>second.xml</loc></sitemap>
+          </sitemapindex>
+        XML
+      )
+      File.write(
+        File.join(dir, "first.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>http://localhost:3000/first</loc></url>
+          </urlset>
+        XML
+      )
+      File.write(
+        File.join(dir, "second.xml"),
+        <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>http://localhost:3000/second</loc></url>
+          </urlset>
+        XML
+      )
+      executor = RecordingExecutor.new
+      parser = Crawlscope::Sitemap.new(path: File.join(dir, "sitemap.xml"), fetch_executor: executor)
+      assert_equal ["http://localhost:3000/first", "http://localhost:3000/second"], parser.urls(base_url: "http://localhost:3000")
+      assert_equal [[File.join(dir, "first.xml"), File.join(dir, "second.xml")]], executor.batches
+    end
+  end
 end

data/test/performance/async_fetch_benchmark.rb ADDED Viewed

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+require "time"
+class DelayedHttpServer
+  attr_reader :base_url
+  def initialize(page_count:, delay_seconds:)
+    @page_count = page_count
+    @delay_seconds = delay_seconds
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+  private
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_xml, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, page_html(path), content_type: "text/html")
+    end
+  ensure
+    socket.close
+  end
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+  def sitemap_xml
+    urls = (1..@page_count).map do |index|
+      "<url><loc>#{@base_url}/pages/#{index}</loc></url>"
+    end.join
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{urls}</urlset>)
+  end
+  def page_html(path)
+    <<~HTML
+      <html>
+        <head><title>#{path}</title></head>
+        <body><main><h1>#{path}</h1><p>#{path} benchmark page</p></main></body>
+      </html>
+    HTML
+  end
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+def measure(name, base_url:, concurrency:, fetch_executor:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  result = Crawlscope::Crawl.new(
+    base_url: base_url,
+    sitemap_path: "#{base_url}/sitemap.xml",
+    rules: [],
+    schema_registry: Crawlscope::SchemaRegistry.default,
+    concurrency: concurrency,
+    fetch_executor: fetch_executor
+  ).call
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  [name, {seconds: elapsed.round(3), pages: result.pages.size, issues: result.issues.size}]
+end
+server = DelayedHttpServer.new(page_count: 24, delay_seconds: 0.08)
+server.start
+begin
+  results = {}
+  [
+    measure("threaded_concurrency_1", base_url: server.base_url, concurrency: 1, fetch_executor: :threaded),
+    measure("threaded_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :threaded),
+    measure("async_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :async)
+  ].each { |name, result| results[name] = result }
+  sequential = results.fetch("threaded_concurrency_1").fetch(:seconds)
+  threaded = results.fetch("threaded_concurrency_8").fetch(:seconds)
+  async = results.fetch("async_concurrency_8").fetch(:seconds)
+  abort "async benchmark failed: async was not meaningfully faster than sequential" unless async < sequential * 0.6
+  abort "async benchmark failed: async was more than 2x slower than threaded" if async > threaded * 2.0
+  puts JSON.pretty_generate(results)
+ensure
+  server.stop
+end

data/test/performance/fetch_executor_matrix.rb ADDED Viewed

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+class MatrixHttpServer
+  attr_reader :base_url
+  def initialize(page_count:, delay_seconds:, link_targets: false)
+    @page_count = page_count
+    @delay_seconds = delay_seconds
+    @link_targets = link_targets
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+  private
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_xml, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, page_html(path), content_type: "text/html")
+    end
+  ensure
+    socket.close
+  end
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+  def sitemap_xml
+    paths = @link_targets ? ["/seed"] : (1..@page_count).map { |index| "/pages/#{index}" }
+    urls = paths.map { |path| "<url><loc>#{@base_url}#{path}</loc></url>" }.join
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{urls}</urlset>)
+  end
+  def page_html(path)
+    links = if @link_targets && path == "/seed"
+      (1..@page_count).map { |index| %(<a href="/targets/#{index}">Target #{index}</a>) }.join
+    else
+      ""
+    end
+    <<~HTML
+      <html>
+        <head>
+          <title>#{path}</title>
+          <meta name="robots" content="noindex">
+        </head>
+        <body>
+          <main><h1>#{path}</h1><p>#{path} benchmark page</p>#{links}</main>
+        </body>
+      </html>
+    HTML
+  end
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+def measure(base_url:, concurrency:, fetch_executor:, rules:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  result = Crawlscope::Crawl.new(
+    base_url: base_url,
+    sitemap_path: "#{base_url}/sitemap.xml",
+    rules: rules,
+    schema_registry: Crawlscope::SchemaRegistry.default,
+    concurrency: concurrency,
+    fetch_executor: fetch_executor
+  ).call
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  {seconds: elapsed, pages: result.pages.size, issues: result.issues.size}
+end
+def median(values)
+  sorted = values.sort
+  sorted[sorted.length / 2]
+end
+def run_case(name:, page_count:, delay_seconds:, concurrency:, link_targets: false)
+  server = MatrixHttpServer.new(page_count: page_count, delay_seconds: delay_seconds, link_targets: link_targets)
+  server.start
+  rules = link_targets ? [Crawlscope::Rules::Links.new] : []
+  threaded = []
+  async = []
+  3.times do
+    threaded << measure(base_url: server.base_url, concurrency: concurrency, fetch_executor: :threaded, rules: rules)
+    async << measure(base_url: server.base_url, concurrency: concurrency, fetch_executor: :async, rules: rules)
+  end
+  threaded_seconds = median(threaded.map { |result| result.fetch(:seconds) })
+  async_seconds = median(async.map { |result| result.fetch(:seconds) })
+  {
+    name: name,
+    page_count: page_count,
+    delay_seconds: delay_seconds,
+    concurrency: concurrency,
+    link_targets: link_targets,
+    threaded_seconds: threaded_seconds.round(3),
+    async_seconds: async_seconds.round(3),
+    async_vs_threaded: (threaded_seconds / async_seconds).round(2),
+    pages: async.first.fetch(:pages),
+    issues: async.first.fetch(:issues)
+  }
+ensure
+  server&.stop
+end
+cases = [
+  {name: "direct_pages_c8", page_count: 48, delay_seconds: 0.02, concurrency: 8},
+  {name: "direct_pages_c16", page_count: 48, delay_seconds: 0.02, concurrency: 16},
+  {name: "slow_direct_pages_c8", page_count: 48, delay_seconds: 0.08, concurrency: 8},
+  {name: "slow_direct_pages_c16", page_count: 48, delay_seconds: 0.08, concurrency: 16},
+  {name: "link_targets_c8", page_count: 48, delay_seconds: 0.02, concurrency: 8, link_targets: true},
+  {name: "slow_link_targets_c8", page_count: 48, delay_seconds: 0.08, concurrency: 8, link_targets: true}
+]
+puts JSON.pretty_generate(cases.map { |attributes| run_case(**attributes) })

data/test/performance/sitemap_expansion_benchmark.rb ADDED Viewed

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+$LOAD_PATH.unshift File.expand_path("../../lib", __dir__)
+require "bundler/setup"
+require "crawlscope"
+require "json"
+require "socket"
+class DelayedSitemapServer
+  attr_reader :base_url
+  def initialize(child_count:, delay_seconds:)
+    @child_count = child_count
+    @delay_seconds = delay_seconds
+    @server = TCPServer.new("127.0.0.1", 0)
+    @base_url = "http://127.0.0.1:#{@server.addr[1]}"
+    @threads = []
+  end
+  def start
+    @thread = Thread.new do
+      loop do
+        socket = @server.accept
+        @threads << Thread.new(socket) { |client| respond(client) }
+      rescue IOError
+        break
+      end
+    end
+  end
+  def stop
+    @server.close
+    @thread&.join
+    @threads.each(&:join)
+  end
+  private
+  def respond(socket)
+    request_line = socket.gets.to_s
+    path = request_line.split[1].to_s
+    read_headers(socket)
+    if path == "/sitemap.xml"
+      write_response(socket, sitemap_index, content_type: "application/xml")
+    else
+      sleep @delay_seconds
+      write_response(socket, child_sitemap(path), content_type: "application/xml")
+    end
+  ensure
+    socket.close
+  end
+  def read_headers(socket)
+    loop do
+      line = socket.gets
+      break if line.nil? || line == "\r\n"
+    end
+  end
+  def sitemap_index
+    children = (1..@child_count).map do |index|
+      "<sitemap><loc>#{@base_url}/sitemaps/#{index}.xml</loc></sitemap>"
+    end.join
+    %(<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#{children}</sitemapindex>)
+  end
+  def child_sitemap(path)
+    index = File.basename(path, ".xml")
+    %(<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>#{@base_url}/pages/#{index}</loc></url></urlset>)
+  end
+  def write_response(socket, body, content_type:)
+    socket.write "HTTP/1.1 200 OK\r\n"
+    socket.write "Content-Type: #{content_type}\r\n"
+    socket.write "Content-Length: #{body.bytesize}\r\n"
+    socket.write "Connection: close\r\n"
+    socket.write "\r\n"
+    socket.write body
+  end
+end
+def measure(name, base_url:, concurrency:, fetch_executor:)
+  started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  urls = Crawlscope::Sitemap.new(
+    path: "#{base_url}/sitemap.xml",
+    concurrency: concurrency,
+    fetch_executor: fetch_executor,
+    timeout_seconds: 5
+  ).urls(base_url: base_url)
+  elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+  [name, {seconds: elapsed.round(3), urls: urls.size}]
+end
+server = DelayedSitemapServer.new(child_count: 24, delay_seconds: 0.08)
+server.start
+begin
+  results = {}
+  [
+    measure("threaded_concurrency_1", base_url: server.base_url, concurrency: 1, fetch_executor: :threaded),
+    measure("threaded_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :threaded),
+    measure("async_concurrency_8", base_url: server.base_url, concurrency: 8, fetch_executor: :async)
+  ].each { |name, result| results[name] = result }
+  sequential = results.fetch("threaded_concurrency_1").fetch(:seconds)
+  threaded = results.fetch("threaded_concurrency_8").fetch(:seconds)
+  async = results.fetch("async_concurrency_8").fetch(:seconds)
+  abort "sitemap benchmark failed: threaded parallelism was not at least 2x faster" unless threaded < sequential * 0.5
+  abort "sitemap benchmark failed: async parallelism was not at least 2x faster" unless async < sequential * 0.5
+  puts JSON.pretty_generate(results)
+ensure
+  server.stop
+end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -24,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: async-http-faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.22'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -200,6 +228,9 @@ files:
 - lib/crawlscope/crawl.rb
 - lib/crawlscope/crawler.rb
 - lib/crawlscope/document_text.rb
+- lib/crawlscope/fetch_executor.rb
+- lib/crawlscope/fetch_executor/async.rb
+- lib/crawlscope/fetch_executor/threaded.rb
 - lib/crawlscope/http.rb
 - lib/crawlscope/issue.rb
 - lib/crawlscope/issue_collection.rb
@@ -234,6 +265,7 @@ files:
 - test/crawlscope/content_quality_rule_test.rb
 - test/crawlscope/crawl_test.rb
 - test/crawlscope/crawler_test.rb
+- test/crawlscope/fetch_executor_test.rb
 - test/crawlscope/http_test.rb
 - test/crawlscope/indexability_rule_test.rb
 - test/crawlscope/links_rule_test.rb
@@ -241,6 +273,7 @@ files:
 - test/crawlscope/metadata_rule_test.rb
 - test/crawlscope/rake_tasks_test.rb
 - test/crawlscope/reporter_test.rb
+- test/crawlscope/result_test.rb
 - test/crawlscope/rule_registry_test.rb
 - test/crawlscope/run_test.rb
 - test/crawlscope/schema_registry_test.rb
@@ -253,6 +286,9 @@ files:
 - test/crawlscope/structured_data_writer_test.rb
 - test/crawlscope/uniqueness_rule_test.rb
 - test/crawlscope/url_test.rb
+- test/performance/async_fetch_benchmark.rb
+- test/performance/fetch_executor_matrix.rb
+- test/performance/sitemap_expansion_benchmark.rb
 - test/release_task_test.rb
 - test/test_helper.rb
 homepage: https://www.ethos-link.com/opensource/crawlscope
@@ -275,7 +311,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="