crawlscope 0.4.0 → 0.6.0

data/lib/crawlscope/rules/uniqueness.rb

@@ -26,11 +26,10 @@ module Crawlscope
       end
 
       def call(urls:, pages:, issues:, context:)
-        page_summaries = pages.filter_map do |page|
-          next unless page.html?
+        @concurrency = context_value(context, :concurrency, default: 1)
+        @fetch_executor = context_value(context, :fetch_executor)
 
-          summary_for(page)
-        end
+        page_summaries = summarize_pages(pages)
 
         validate_duplicates(page_summaries, issues)
         validate_near_duplicates(page_summaries, issues)
@@ -58,6 +57,7 @@ module Crawlscope
 
         {
           content_fingerprint_digest: content_fingerprint_digest(page.doc),
+          canonical: page.doc.at_css('link[rel="canonical"]')&.[]("href").to_s.strip,
           description: page.doc.at_css('meta[name="description"]')&.[]("content").to_s.strip,
           shingles: shingles_for(tokens),
           title: page.doc.at_css("title")&.text.to_s.strip,
@@ -65,6 +65,16 @@ module Crawlscope
         }
       end
 
+      def summarize_pages(pages)
+        html_pages = pages.select(&:html?)
+
+        FetchExecutor.map(
+          name: @fetch_executor,
+          concurrency: @concurrency,
+          items: html_pages
+        ) { |page| summary_for(page) }
+      end
+
       def validate_duplicates(page_summaries, issues)
         duplicates_for(page_summaries, :title).each do |value, urls|
           issues.add(
@@ -98,6 +108,27 @@ module Crawlscope
             details: {urls: urls}
           )
         end
+
+        duplicate_content_clusters_without_canonical(page_summaries).each do |urls|
+          issues.add(
+            code: :duplicate_pages_without_canonical,
+            severity: :warning,
+            category: :uniqueness,
+            url: nil,
+            message: "duplicate pages without canonical => #{urls.join(", ")}",
+            details: {urls: urls}
+          )
+        end
+      end
+
+      def duplicate_content_clusters_without_canonical(page_summaries)
+        page_summaries
+          .select { |page| !page[:content_fingerprint_digest].nil? }
+          .group_by { |page| page[:content_fingerprint_digest] }
+          .values
+          .select { |pages| pages.size > 1 }
+          .select { |pages| pages.any? { |page| page[:canonical].to_s.empty? } }
+          .map { |pages| pages.map { |page| page[:url] } }
       end
 
       def shingles_for(tokens)
@@ -155,6 +186,16 @@ module Crawlscope
 
         intersection_size.to_f / smaller_set_size
       end
+
+      def context_value(context, name, default: nil)
+        if context.respond_to?(name)
+          context.public_send(name)
+        elsif context.respond_to?(:key?) && context.key?(name)
+          context[name]
+        else
+          default
+        end
+      end
     end
   end
 end

data/lib/crawlscope/sitemap.rb

@@ -9,28 +9,45 @@ module Crawlscope
   class Sitemap
     SITEMAP_NAMESPACE = {"xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9"}.freeze
 
-    def initialize(path:)
+    def initialize(path:, adapter: nil, concurrency: Configuration::DEFAULT_CONCURRENCY, fetch_executor: Configuration::DEFAULT_FETCH_EXECUTOR, timeout_seconds: Configuration::DEFAULT_TIMEOUT_SECONDS)
       @path = path
+      @adapter = adapter
+      @concurrency = concurrency
+      @fetch_executor = fetch_executor
+      @timeout_seconds = timeout_seconds
     end
 
     def urls(base_url:)
-      collect_urls(@path, base_url: base_url, visited: Set.new).uniq
+      collect_urls(@path, base_url: base_url, visited: Set.new, visited_mutex: Mutex.new).uniq
     end
 
     private
 
-    def collect_urls(source, base_url:, visited:)
-      return [] if visited.include?(source)
+    def collect_urls(source, base_url:, visited:, visited_mutex:)
+      already_visited = visited_mutex.synchronize do
+        if visited.include?(source)
+          true
+        else
+          visited.add(source)
+          false
+        end
+      end
+      return [] if already_visited
 
-      visited.add(source)
       document = Nokogiri::XML(read(source))
       root_name = document.root&.name
+      unless %w[sitemapindex urlset].include?(root_name)
+        raise ValidationError, "Sitemap #{source} has unexpected root #{root_name.inspect}"
+      end
 
       if root_name == "sitemapindex"
-        document.xpath("//xmlns:sitemap/xmlns:loc", SITEMAP_NAMESPACE).flat_map do |node|
-          child_source = resolve_child_source(source, node.text.to_s.strip, base_url: base_url)
-          collect_urls(child_source, base_url: base_url, visited: visited)
+        child_sources = document.xpath("//xmlns:sitemap/xmlns:loc", SITEMAP_NAMESPACE).map do |node|
+          resolve_child_source(source, node.text.to_s.strip, base_url: base_url)
         end
+
+        fetch_executor.call(child_sources) do |child_source|
+          collect_urls(child_source, base_url: base_url, visited: visited, visited_mutex: visited_mutex)
+        end.flatten
       else
         document.xpath("//xmlns:url/xmlns:loc", SITEMAP_NAMESPACE).map do |node|
           Url.normalize_for_base(node.text.to_s.strip, base_url: base_url)
@@ -40,7 +57,12 @@ module Crawlscope
 
     def read(source)
       if Url.remote?(source)
-        connection.get(source).body
+        response = connection.get(source)
+        unless response.status.to_i.between?(200, 299)
+          raise ValidationError, "Sitemap #{source} returned HTTP #{response.status}"
+        end
+
+        response.body
       else
         File.read(source)
       end
@@ -69,11 +91,16 @@ module Crawlscope
     end
 
     def connection
-      @connection ||= Faraday.new do |faraday|
+      Faraday.new do |faraday|
         faraday.response :follow_redirects, limit: Http::MAX_REDIRECTS
-        faraday.options.timeout = 20
-        faraday.options.open_timeout = 20
+        faraday.options.timeout = @timeout_seconds
+        faraday.options.open_timeout = @timeout_seconds
+        faraday.adapter @adapter if @adapter
       end
     end
+
+    def fetch_executor
+      @fetch_executor_instance ||= FetchExecutor.build(name: @fetch_executor, concurrency: @concurrency)
+    end
   end
 end

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

data/lib/tasks/crawlscope_tasks.rake

@@ -1,43 +1,43 @@
 namespace :crawlscope do
-  desc "Validate URLs with all default Crawlscope rules. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY"
-  task validate: :environment do
-    Crawlscope::RakeTasks.validate
+  desc "Validate URLs with all default Crawlscope rules. Args: [url,sitemap,rules]. ENV: URL, SITEMAP, RULES, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, CONCURRENCY, FETCH_EXECUTOR"
+  task :validate, [:url, :sitemap, :rules] => :environment do |_task, args|
+    Crawlscope::RakeTasks.validate(url: args[:url], sitemap_path: args[:sitemap], rule_names: args[:rules])
   end
 
   namespace :validate do
-    desc "Directly validate JSON-LD on one or more URLs. ENV: URL (semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
-    task ldjson: :environment do
-      Crawlscope::RakeTasks.ldjson
+    desc "Directly validate JSON-LD on one URL. Args: [url]. ENV: URL (semicolon-separated), DEBUG=1, JS=1, TIMEOUT, NETWORK_IDLE_TIMEOUT, REPORT_PATH, SUMMARY=1"
+    task :ldjson, [:url] => :environment do |_task, args|
+      Crawlscope::RakeTasks.ldjson(urls: args[:url])
     end
 
-    desc "Validate URLs with the indexability rule. ENV: URL, SITEMAP, JS=1"
-    task indexability: :environment do
-      Crawlscope::RakeTasks.validate_rule("indexability")
+    desc "Validate URLs with the indexability rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :indexability, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("indexability", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the metadata rule. ENV: URL, SITEMAP, JS=1"
-    task metadata: :environment do
-      Crawlscope::RakeTasks.validate_rule("metadata")
+    desc "Validate URLs with the metadata rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :metadata, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("metadata", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate sitemap URLs with the structured_data rule. ENV: URL, SITEMAP, JS=1"
-    task structured_data: :environment do
-      Crawlscope::RakeTasks.validate_rule("structured_data")
+    desc "Validate sitemap URLs with the structured_data rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :structured_data, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("structured_data", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the uniqueness rule. ENV: URL, SITEMAP, JS=1"
-    task uniqueness: :environment do
-      Crawlscope::RakeTasks.validate_rule("uniqueness")
+    desc "Validate URLs with the uniqueness rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :uniqueness, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("uniqueness", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the content_quality rule. ENV: URL, SITEMAP, JS=1"
-    task content_quality: :environment do
-      Crawlscope::RakeTasks.validate_rule("content_quality")
+    desc "Validate URLs with the content_quality rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :content_quality, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("content_quality", url: args[:url], sitemap_path: args[:sitemap])
     end
 
-    desc "Validate URLs with the links rule. ENV: URL, SITEMAP, JS=1"
-    task links: :environment do
-      Crawlscope::RakeTasks.validate_rule("links")
+    desc "Validate URLs with the links rule. Args: [url,sitemap]. ENV: URL, SITEMAP, JS=1"
+    task :links, [:url, :sitemap] => :environment do |_task, args|
+      Crawlscope::RakeTasks.validate_rule("links", url: args[:url], sitemap_path: args[:sitemap])
     end
   end
 end

data/test/crawlscope/cli_test.rb

@@ -4,11 +4,12 @@ require "test_helper"
 
 class CrawlscopeCliTest < Minitest::Test
   class FakeConfiguration
-    attr_accessor :base_url, :concurrency, :network_idle_timeout_seconds, :output, :renderer, :timeout_seconds
+    attr_accessor :base_url, :concurrency, :fetch_executor, :network_idle_timeout_seconds, :output, :renderer, :timeout_seconds
 
     def initialize
       @base_url = nil
       @concurrency = 10
+      @fetch_executor = :async
       @network_idle_timeout_seconds = 5
       @renderer = :http
       @timeout_seconds = 20
@@ -95,7 +96,7 @@ class CrawlscopeCliTest < Minitest::Test
     err = StringIO.new
 
     status = Crawlscope::Cli.start(
-      ["validate", "--url", "https://example.com", "--sitemap", "https://example.com/sitemap-pages.xml", "--rules", "metadata,links", "--renderer", "browser", "--timeout", "30", "--network-idle-timeout", "9", "--concurrency", "3"],
+      ["validate", "--url", "https://example.com", "--sitemap", "https://example.com/sitemap-pages.xml", "--rules", "metadata,links", "--renderer", "browser", "--timeout", "30", "--network-idle-timeout", "9", "--concurrency", "3", "--fetch-executor", "async"],
       out: out,
       err: err,
       configuration: configuration,
@@ -115,10 +116,35 @@ class CrawlscopeCliTest < Minitest::Test
     assert_equal 30, configuration.timeout_seconds
     assert_equal 9, configuration.network_idle_timeout_seconds
     assert_equal 3, configuration.concurrency
+    assert_equal "async", configuration.fetch_executor
     assert_same out, configuration.output
     assert_empty err.string
   end
 
+  def test_validate_reads_fetch_executor_from_environment
+    configuration = FakeConfiguration.new
+    task = FakeTask.new
+
+    with_env("FETCH_EXECUTOR" => "async") do
+      status = Crawlscope::Cli.start(["validate", "--url", "https://example.com"], out: StringIO.new, err: StringIO.new, configuration: configuration, task: task)
+
+      assert_equal 0, status
+    end
+
+    assert_equal "async", configuration.fetch_executor
+  end
+
+  def test_validate_uses_threaded_executor_for_browser_rendering_by_default
+    configuration = FakeConfiguration.new
+    task = FakeTask.new
+
+    status = Crawlscope::Cli.start(["validate", "--url", "https://example.com", "--renderer", "browser"], out: StringIO.new, err: StringIO.new, configuration: configuration, task: task)
+
+    assert_equal 0, status
+    assert_equal :browser, configuration.renderer
+    assert_equal :threaded, configuration.fetch_executor
+  end
+
   def test_ldjson_reads_urls_from_environment
     configuration = FakeConfiguration.new
     task = FakeTask.new
@@ -265,6 +291,7 @@ class CrawlscopeCliTest < Minitest::Test
 
     assert_equal 1, status
     assert_includes err.string, "No URLs found in sitemap"
+    refute_includes err.string, "Usage:"
   end
 
   private

data/test/crawlscope/configuration_test.rb

@@ -13,6 +13,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
       config.sitemap_path = -> { "/tmp/sitemap.xml" }
       config.site_name = -> { "Example" }
       config.concurrency = -> { 4 }
+      config.fetch_executor = -> { :threaded }
     end
 
     audit = Crawlscope.configuration.audit
@@ -20,6 +21,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "https://example.com", audit.instance_variable_get(:@base_url)
     assert_equal "/tmp/sitemap.xml", audit.instance_variable_get(:@sitemap_path)
     assert_equal 4, audit.instance_variable_get(:@concurrency)
+    assert_equal :threaded, audit.instance_variable_get(:@fetch_executor)
     assert_equal %i[
       indexability
       metadata
@@ -55,6 +57,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
 
     assert_equal [200, 301, 302], config.allowed_statuses
     assert_equal 10, config.concurrency
+    assert_equal :async, config.fetch_executor
     assert_equal 4, config.browser_concurrency
     assert_equal 5, config.network_idle_timeout_seconds
     assert_equal :http, config.renderer
@@ -63,10 +66,18 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert config.scroll_page?
   end
 
+  def test_browser_renderer_defaults_to_threaded_fetch_executor
+    config = Crawlscope::Configuration.new
+    config.renderer = :browser
+
+    assert_equal :threaded, config.fetch_executor
+  end
+
   def test_configured_values_are_normalized
     config = Crawlscope::Configuration.new
     config.allowed_statuses = ["200", "404"]
     config.concurrency = "2"
+    config.fetch_executor = "async"
     config.network_idle_timeout_seconds = "7"
     config.renderer = "browser"
     config.timeout_seconds = "9"
@@ -74,6 +85,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
 
     assert_equal [200, 404], config.allowed_statuses
     assert_equal 2, config.concurrency
+    assert_equal :async, config.fetch_executor
     assert_equal 2, config.browser_concurrency
     assert_equal 7, config.network_idle_timeout_seconds
     assert_equal :browser, config.renderer
@@ -90,6 +102,15 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "Crawlscope renderer must be http or browser", error.message
   end
 
+  def test_fetch_executor_must_be_supported
+    config = Crawlscope::Configuration.new
+    config.fetch_executor = "processes"
+
+    error = assert_raises(Crawlscope::ConfigurationError) { config.fetch_executor }
+
+    assert_equal "Crawlscope fetch_executor must be threaded or async", error.message
+  end
+
   def test_numeric_values_must_be_positive_integers
     config = Crawlscope::Configuration.new
     config.concurrency = "0"

data/test/crawlscope/content_quality_rule_test.rb

@@ -27,6 +27,24 @@ class CrawlscopeContentQualityRuleTest < Minitest::Test
     refute_includes issues.to_a.map(&:code), :low_visible_text_ratio
   end
 
+  def test_visible_text_ratio_ignores_form_payload_markup
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      main: <<~HTML
+        <p>#{Array.new(260) { |index| "word#{index}" }.join(" ")}</p>
+        <form>
+          <div data-select-autocomplete-options-value="#{"x" * 50_000}">
+            <input type="text" name="country">
+          </div>
+        </form>
+      HTML
+    )
+
+    Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    refute_includes issues.to_a.map(&:code), :low_visible_text_ratio
+  end
+
   def test_reports_low_unique_token_ratio_for_repetitive_content
     issues = Crawlscope::IssueCollection.new
     page = page_with(main: ("hotel location service " * 100).strip)

data/test/crawlscope/crawl_test.rb

@@ -3,6 +3,36 @@
 require "test_helper"
 
 class CrawlscopeCrawlTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :batches
+
+    def initialize
+      @batches = []
+    end
+
+    def call(urls)
+      @batches << urls
+      urls.map { |url| yield(url) }
+    end
+  end
+
+  class PageMapFetcher
+    attr_reader :closed
+
+    def initialize(pages)
+      @pages = pages
+      @closed = false
+    end
+
+    def close
+      @closed = true
+    end
+
+    def fetch(url)
+      @pages.fetch(url)
+    end
+  end
+
   def setup
     @tmp_dir = Dir.mktmpdir
     @sitemap_path = File.join(@tmp_dir, "sitemap.xml")
@@ -12,6 +42,29 @@ class CrawlscopeCrawlTest < Minitest::Test
     FileUtils.rm_rf(@tmp_dir)
   end
 
+  def test_http_renderer_defaults_to_async_executor
+    crawl = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default
+    )
+
+    assert_equal :async, crawl.instance_variable_get(:@fetch_executor)
+  end
+
+  def test_browser_renderer_defaults_to_threaded_executor
+    crawl = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      renderer: :browser
+    )
+
+    assert_equal :threaded, crawl.instance_variable_get(:@fetch_executor)
+  end
+
   def test_returns_ok_when_metadata_is_valid
     File.write(
       @sitemap_path,
@@ -56,7 +109,8 @@ class CrawlscopeCrawlTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: @sitemap_path,
       rules: Crawlscope::RuleRegistry.default(site_name: "Example").rules,
-      schema_registry: Crawlscope::SchemaRegistry.default
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
     ).call
 
     assert result.ok?
@@ -97,10 +151,11 @@ class CrawlscopeCrawlTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: @sitemap_path,
       rules: Crawlscope::RuleRegistry.default(site_name: "Example").rules,
-      schema_registry: Crawlscope::SchemaRegistry.default
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
     ).call
 
-    refute result.ok?
+    assert result.ok?
     assert_equal %i[
       incomplete_open_graph_tags
       meta_description_too_long
@@ -188,4 +243,113 @@ class CrawlscopeCrawlTest < Minitest::Test
     assert_equal ["https://example.com/pricing"], fake_browser.urls
     assert fake_browser.closed
   end
+
+  def test_async_executor_requires_http_renderer
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/pricing</loc></url>
+        </urlset>
+      XML
+    )
+
+    error = assert_raises(Crawlscope::ConfigurationError) do
+      Crawlscope::Crawl.new(
+        base_url: "https://example.com",
+        sitemap_path: @sitemap_path,
+        rules: [],
+        schema_registry: Crawlscope::SchemaRegistry.default,
+        renderer: :browser,
+        fetch_executor: :async
+      ).call
+    end
+
+    assert_equal "Async fetch execution is only supported with http rendering", error.message
+  end
+
+  def test_reports_sitemap_redirect_url
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/old</loc></url>
+        </urlset>
+      XML
+    )
+
+    stub_request(:get, "https://example.com/old")
+      .to_return(status: 301, headers: {"Location" => "https://example.com/new"}, body: "")
+    stub_request(:get, "https://example.com/new")
+      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<html><body>Moved</body></html>")
+
+    result = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
+    ).call
+
+    assert_includes result.issues.to_a.map(&:code), :sitemap_redirect_url
+  end
+
+  def test_resolves_uncrawled_link_targets_as_a_bounded_batch
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/guide</loc></url>
+        </urlset>
+      XML
+    )
+
+    executor = RecordingExecutor.new
+    fetcher = PageMapFetcher.new(
+      "https://example.com/guide" => page(
+        "https://example.com/guide",
+        "<main><a href=\"/one\">One</a><a href=\"/two\">Two</a></main>"
+      ),
+      "https://example.com/one" => page("https://example.com/one", "<main>One</main>"),
+      "https://example.com/two" => page("https://example.com/two", "<main>Two</main>")
+    )
+
+    Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [Crawlscope::Rules::Links.new],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      renderer: :browser,
+      browser_factory: -> { fetcher },
+      fetch_executor: executor,
+      concurrency: 2
+    ).call
+
+    assert_equal(
+      [
+        ["https://example.com/guide"],
+        ["https://example.com/one", "https://example.com/two"]
+      ],
+      executor.batches
+    )
+    assert fetcher.closed
+  end
+
+  private
+
+  def page(url, body)
+    Crawlscope::Page.new(
+      url: url,
+      normalized_url: url,
+      final_url: url,
+      normalized_final_url: url,
+      status: 200,
+      headers: {"content-type" => "text/html"},
+      body: body,
+      doc: Nokogiri::HTML(body)
+    )
+  end
 end

data/test/crawlscope/crawler_test.rb

@@ -31,4 +31,65 @@ class CrawlscopeCrawlerTest < Minitest::Test
     assert_nil error_page.status
     assert_equal "Timeout::Error: fetch timed out", error_page.error
   end
+
+  def test_preserves_input_order
+    pages = Crawlscope::Crawler.new(page_fetcher: RaisingFetcher.new, concurrency: 2).call(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"]
+    )
+
+    assert_equal(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"],
+      pages.map(&:url)
+    )
+  end
+
+  class AsyncFetcher
+    attr_reader :active_fetches
+
+    def initialize
+      @active_fetches = 0
+      @max_active_fetches = 0
+      @mutex = Mutex.new
+    end
+
+    def fetch(url)
+      @mutex.synchronize do
+        @active_fetches += 1
+        @max_active_fetches = [@max_active_fetches, @active_fetches].max
+      end
+
+      Async::Task.current.sleep(0.01)
+
+      Crawlscope::Page.new(
+        url: url,
+        normalized_url: url,
+        final_url: url,
+        normalized_final_url: url,
+        status: 200,
+        headers: {},
+        body: "<html></html>",
+        doc: Nokogiri::HTML("<html></html>")
+      )
+    ensure
+      @mutex.synchronize { @active_fetches -= 1 }
+    end
+
+    def max_active_fetches
+      @mutex.synchronize { @max_active_fetches }
+    end
+  end
+
+  def test_async_executor_respects_concurrency_and_preserves_order
+    fetcher = AsyncFetcher.new
+
+    pages = Crawlscope::Crawler.new(page_fetcher: fetcher, concurrency: 2, fetch_executor: :async).call(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"]
+    )
+
+    assert_equal(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"],
+      pages.map(&:url)
+    )
+    assert_operator fetcher.max_active_fetches, :<=, 2
+  end
 end