crawlscope 0.5.0 → 0.7.0

data/test/crawlscope/configuration_test.rb

@@ -13,6 +13,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
       config.sitemap_path = -> { "/tmp/sitemap.xml" }
       config.site_name = -> { "Example" }
       config.concurrency = -> { 4 }
+      config.fetch_executor = -> { :threaded }
     end
 
     audit = Crawlscope.configuration.audit
@@ -20,6 +21,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "https://example.com", audit.instance_variable_get(:@base_url)
     assert_equal "/tmp/sitemap.xml", audit.instance_variable_get(:@sitemap_path)
     assert_equal 4, audit.instance_variable_get(:@concurrency)
+    assert_equal :threaded, audit.instance_variable_get(:@fetch_executor)
     assert_equal %i[
       indexability
       metadata
@@ -55,6 +57,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
 
     assert_equal [200, 301, 302], config.allowed_statuses
     assert_equal 10, config.concurrency
+    assert_equal :async, config.fetch_executor
     assert_equal 4, config.browser_concurrency
     assert_equal 5, config.network_idle_timeout_seconds
     assert_equal :http, config.renderer
@@ -63,10 +66,18 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert config.scroll_page?
   end
 
+  def test_browser_renderer_defaults_to_threaded_fetch_executor
+    config = Crawlscope::Configuration.new
+    config.renderer = :browser
+
+    assert_equal :threaded, config.fetch_executor
+  end
+
   def test_configured_values_are_normalized
     config = Crawlscope::Configuration.new
     config.allowed_statuses = ["200", "404"]
     config.concurrency = "2"
+    config.fetch_executor = "async"
     config.network_idle_timeout_seconds = "7"
     config.renderer = "browser"
     config.timeout_seconds = "9"
@@ -74,6 +85,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
 
     assert_equal [200, 404], config.allowed_statuses
     assert_equal 2, config.concurrency
+    assert_equal :async, config.fetch_executor
     assert_equal 2, config.browser_concurrency
     assert_equal 7, config.network_idle_timeout_seconds
     assert_equal :browser, config.renderer
@@ -90,6 +102,15 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "Crawlscope renderer must be http or browser", error.message
   end
 
+  def test_fetch_executor_must_be_supported
+    config = Crawlscope::Configuration.new
+    config.fetch_executor = "processes"
+
+    error = assert_raises(Crawlscope::ConfigurationError) { config.fetch_executor }
+
+    assert_equal "Crawlscope fetch_executor must be threaded or async", error.message
+  end
+
   def test_numeric_values_must_be_positive_integers
     config = Crawlscope::Configuration.new
     config.concurrency = "0"

data/test/crawlscope/content_quality_rule_test.rb

@@ -27,6 +27,24 @@ class CrawlscopeContentQualityRuleTest < Minitest::Test
     refute_includes issues.to_a.map(&:code), :low_visible_text_ratio
   end
 
+  def test_visible_text_ratio_ignores_form_payload_markup
+    issues = Crawlscope::IssueCollection.new
+    page = page_with(
+      main: <<~HTML
+        <p>#{Array.new(260) { |index| "word#{index}" }.join(" ")}</p>
+        <form>
+          <div data-select-autocomplete-options-value="#{"x" * 50_000}">
+            <input type="text" name="country">
+          </div>
+        </form>
+      HTML
+    )
+
+    Crawlscope::Rules::ContentQuality.new.call(urls: [page.url], pages: [page], issues: issues)
+
+    refute_includes issues.to_a.map(&:code), :low_visible_text_ratio
+  end
+
   def test_reports_low_unique_token_ratio_for_repetitive_content
     issues = Crawlscope::IssueCollection.new
     page = page_with(main: ("hotel location service " * 100).strip)

data/test/crawlscope/crawl_test.rb

@@ -3,6 +3,36 @@
 require "test_helper"
 
 class CrawlscopeCrawlTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :batches
+
+    def initialize
+      @batches = []
+    end
+
+    def call(urls)
+      @batches << urls
+      urls.map { |url| yield(url) }
+    end
+  end
+
+  class PageMapFetcher
+    attr_reader :closed
+
+    def initialize(pages)
+      @pages = pages
+      @closed = false
+    end
+
+    def close
+      @closed = true
+    end
+
+    def fetch(url)
+      @pages.fetch(url)
+    end
+  end
+
   def setup
     @tmp_dir = Dir.mktmpdir
     @sitemap_path = File.join(@tmp_dir, "sitemap.xml")
@@ -12,6 +42,29 @@ class CrawlscopeCrawlTest < Minitest::Test
     FileUtils.rm_rf(@tmp_dir)
   end
 
+  def test_http_renderer_defaults_to_async_executor
+    crawl = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default
+    )
+
+    assert_equal :async, crawl.instance_variable_get(:@fetch_executor)
+  end
+
+  def test_browser_renderer_defaults_to_threaded_executor
+    crawl = Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      renderer: :browser
+    )
+
+    assert_equal :threaded, crawl.instance_variable_get(:@fetch_executor)
+  end
+
   def test_returns_ok_when_metadata_is_valid
     File.write(
       @sitemap_path,
@@ -56,7 +109,8 @@ class CrawlscopeCrawlTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: @sitemap_path,
       rules: Crawlscope::RuleRegistry.default(site_name: "Example").rules,
-      schema_registry: Crawlscope::SchemaRegistry.default
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
     ).call
 
     assert result.ok?
@@ -97,10 +151,11 @@ class CrawlscopeCrawlTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: @sitemap_path,
       rules: Crawlscope::RuleRegistry.default(site_name: "Example").rules,
-      schema_registry: Crawlscope::SchemaRegistry.default
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
     ).call
 
-    refute result.ok?
+    assert result.ok?
     assert_equal %i[
       incomplete_open_graph_tags
       meta_description_too_long
@@ -189,6 +244,31 @@ class CrawlscopeCrawlTest < Minitest::Test
     assert fake_browser.closed
   end
 
+  def test_async_executor_requires_http_renderer
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/pricing</loc></url>
+        </urlset>
+      XML
+    )
+
+    error = assert_raises(Crawlscope::ConfigurationError) do
+      Crawlscope::Crawl.new(
+        base_url: "https://example.com",
+        sitemap_path: @sitemap_path,
+        rules: [],
+        schema_registry: Crawlscope::SchemaRegistry.default,
+        renderer: :browser,
+        fetch_executor: :async
+      ).call
+    end
+
+    assert_equal "Async fetch execution is only supported with http rendering", error.message
+  end
+
   def test_reports_sitemap_redirect_url
     File.write(
       @sitemap_path,
@@ -209,9 +289,67 @@ class CrawlscopeCrawlTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: @sitemap_path,
       rules: [],
-      schema_registry: Crawlscope::SchemaRegistry.default
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded
     ).call
 
     assert_includes result.issues.to_a.map(&:code), :sitemap_redirect_url
   end
+
+  def test_resolves_uncrawled_link_targets_as_a_bounded_batch
+    File.write(
+      @sitemap_path,
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+          <url><loc>https://example.com/guide</loc></url>
+        </urlset>
+      XML
+    )
+
+    executor = RecordingExecutor.new
+    fetcher = PageMapFetcher.new(
+      "https://example.com/guide" => page(
+        "https://example.com/guide",
+        "<main><a href=\"/one\">One</a><a href=\"/two\">Two</a></main>"
+      ),
+      "https://example.com/one" => page("https://example.com/one", "<main>One</main>"),
+      "https://example.com/two" => page("https://example.com/two", "<main>Two</main>")
+    )
+
+    Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: @sitemap_path,
+      rules: [Crawlscope::Rules::Links.new],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      renderer: :browser,
+      browser_factory: -> { fetcher },
+      fetch_executor: executor,
+      concurrency: 2
+    ).call
+
+    assert_equal(
+      [
+        ["https://example.com/guide"],
+        ["https://example.com/one", "https://example.com/two"]
+      ],
+      executor.batches
+    )
+    assert fetcher.closed
+  end
+
+  private
+
+  def page(url, body)
+    Crawlscope::Page.new(
+      url: url,
+      normalized_url: url,
+      final_url: url,
+      normalized_final_url: url,
+      status: 200,
+      headers: {"content-type" => "text/html"},
+      body: body,
+      doc: Nokogiri::HTML(body)
+    )
+  end
 end

data/test/crawlscope/crawler_test.rb

@@ -31,4 +31,65 @@ class CrawlscopeCrawlerTest < Minitest::Test
     assert_nil error_page.status
     assert_equal "Timeout::Error: fetch timed out", error_page.error
   end
+
+  def test_preserves_input_order
+    pages = Crawlscope::Crawler.new(page_fetcher: RaisingFetcher.new, concurrency: 2).call(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"]
+    )
+
+    assert_equal(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"],
+      pages.map(&:url)
+    )
+  end
+
+  class AsyncFetcher
+    attr_reader :active_fetches
+
+    def initialize
+      @active_fetches = 0
+      @max_active_fetches = 0
+      @mutex = Mutex.new
+    end
+
+    def fetch(url)
+      @mutex.synchronize do
+        @active_fetches += 1
+        @max_active_fetches = [@max_active_fetches, @active_fetches].max
+      end
+
+      Async::Task.current.sleep(0.01)
+
+      Crawlscope::Page.new(
+        url: url,
+        normalized_url: url,
+        final_url: url,
+        normalized_final_url: url,
+        status: 200,
+        headers: {},
+        body: "<html></html>",
+        doc: Nokogiri::HTML("<html></html>")
+      )
+    ensure
+      @mutex.synchronize { @active_fetches -= 1 }
+    end
+
+    def max_active_fetches
+      @mutex.synchronize { @max_active_fetches }
+    end
+  end
+
+  def test_async_executor_respects_concurrency_and_preserves_order
+    fetcher = AsyncFetcher.new
+
+    pages = Crawlscope::Crawler.new(page_fetcher: fetcher, concurrency: 2, fetch_executor: :async).call(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"]
+    )
+
+    assert_equal(
+      ["https://example.com/one", "https://example.com/two", "https://example.com/three"],
+      pages.map(&:url)
+    )
+    assert_operator fetcher.max_active_fetches, :<=, 2
+  end
 end

data/test/crawlscope/fetch_executor_test.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeFetchExecutorTest < Minitest::Test
+  class RecordingExecutor
+    attr_reader :items
+
+    def call(items)
+      @items = items
+      items.map { |item| yield(item) }
+    end
+  end
+
+  def test_map_preserves_input_order
+    results = Crawlscope::FetchExecutor.map(name: :threaded, concurrency: 2, items: [3, 1, 2]) do |item|
+      item * 10
+    end
+
+    assert_equal [30, 10, 20], results
+  end
+
+  def test_map_uses_sequential_fallback_for_single_item
+    executor = RecordingExecutor.new
+
+    results = Crawlscope::FetchExecutor.map(name: executor, concurrency: 4, items: ["one"]) do |item|
+      item.upcase
+    end
+
+    assert_equal ["ONE"], results
+    assert_nil executor.items
+  end
+
+  def test_map_uses_injected_executor_for_parallel_work
+    executor = RecordingExecutor.new
+
+    results = Crawlscope::FetchExecutor.map(name: executor, concurrency: 4, items: %w[a b]) do |item|
+      item.upcase
+    end
+
+    assert_equal %w[A B], results
+    assert_equal %w[a b], executor.items
+  end
+end

data/test/crawlscope/links_rule_test.rb

@@ -218,6 +218,33 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     assert_includes codes, :canonical_points_to_redirect
   end
 
+  def test_does_not_report_missing_inlinks_for_root_canonical
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {crawled: true, error: nil, final_url: target_url, html: true, status: 200}
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/", "https://example.com/about"],
+      pages: [
+        page(
+          url: "https://example.com/",
+          body: <<~HTML
+            <html>
+              <head><link rel="canonical" href="https://example.com/"></head>
+              <body><main><a href="/about">About</a></main></body>
+            </html>
+          HTML
+        ),
+        page(url: "https://example.com/about", body: "<main><a href=\"/\">Home</a></main>")
+      ],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    refute_includes issues.to_a.map(&:code), :canonical_no_internal_inlinks
+  end
+
   def test_reports_indexable_internal_pages_missing_from_sitemap
     issues = Crawlscope::IssueCollection.new
     resolver = lambda do |target_url|
@@ -242,6 +269,54 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     assert_equal "https://example.com/hidden", issue.url
   end
 
+  def test_does_not_report_noindex_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        doc: Nokogiri::HTML("<head><meta name=\"robots\" content=\"noindex, follow\"></head>"),
+        error: nil,
+        final_url: target_url,
+        headers: {},
+        html: true,
+        status: 200
+      }
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    refute_includes issues.to_a.map(&:code), :indexable_page_missing_from_sitemap
+  end
+
+  def test_does_not_report_x_robots_noindex_internal_pages_missing_from_sitemap
+    issues = Crawlscope::IssueCollection.new
+    resolver = lambda do |target_url|
+      {
+        crawled: false,
+        doc: Nokogiri::HTML("<main>Hidden</main>"),
+        error: nil,
+        final_url: target_url,
+        headers: {"X-Robots-Tag" => "noindex"},
+        html: true,
+        status: 200
+      }
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    refute_includes issues.to_a.map(&:code), :indexable_page_missing_from_sitemap
+  end
+
   def test_reports_url_hygiene_issues
     issues = Crawlscope::IssueCollection.new
     long_path = "a" * 2_050
@@ -300,6 +375,32 @@ class CrawlscopeLinksRuleTest < Minitest::Test
     assert_includes redirect_issue.message, "https://example.com/pricing"
   end
 
+  def test_reuses_link_target_resolution_for_later_link_checks
+    issues = Crawlscope::IssueCollection.new
+    resolution_counts = Hash.new(0)
+    resolver = lambda do |target_url|
+      resolution_counts[target_url] += 1
+      {
+        crawled: false,
+        doc: Nokogiri::HTML("<main>Hidden</main>"),
+        error: nil,
+        final_url: target_url,
+        headers: {},
+        html: true,
+        status: 200
+      }
+    end
+
+    Crawlscope::Rules::Links.new.call(
+      urls: ["https://example.com/guide"],
+      pages: [page(url: "https://example.com/guide", body: "<main><a href=\"/hidden\">Hidden</a></main>")],
+      issues: issues,
+      context: context(resolver: resolver)
+    )
+
+    assert_equal 1, resolution_counts.fetch("https://example.com/hidden")
+  end
+
   def test_ignores_links_that_should_not_be_crawled
     issues = Crawlscope::IssueCollection.new
 

data/test/crawlscope/loader_test.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "test_helper"
+require "open3"
+require "rbconfig"
 
 class CrawlscopeLoaderTest < Minitest::Test
   def test_eager_loads_cleanly
@@ -8,4 +10,39 @@ class CrawlscopeLoaderTest < Minitest::Test
       Crawlscope.loader.eager_load
     end
   end
+
+  def test_entrypoint_does_not_load_crawl_dependencies
+    script = <<~RUBY
+      require "crawlscope"
+
+      loaded = $LOADED_FEATURES.grep(%r{/(?:async|faraday|json-schema|nokogiri)(?:/|\\.rb)})
+      abort loaded.join("\\n") unless loaded.empty?
+    RUBY
+
+    _stdout, stderr, status = Open3.capture3(
+      RbConfig.ruby,
+      "-I#{File.expand_path("../../lib", __dir__)}",
+      "-e",
+      script
+    )
+
+    assert status.success?, stderr
+  end
+
+  def test_task_entrypoint_registers_tasks
+    script = <<~RUBY
+      require "crawlscope/tasks"
+
+      abort "task missing" unless Rake::Task.task_defined?("crawlscope:validate")
+    RUBY
+
+    _stdout, stderr, status = Open3.capture3(
+      RbConfig.ruby,
+      "-I#{File.expand_path("../../lib", __dir__)}",
+      "-e",
+      script
+    )
+
+    assert status.success?, stderr
+  end
 end