crawlscope 0.7.2 → 0.8.0

data/lib/crawlscope/server_timing.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class ServerTiming
+    include Enumerable
+
+    TOKEN = /\A[!#$%&'*+\-.^_`|~0-9A-Za-z]+\z/
+    NUMBER = /\A-?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[eE][+-]?\d+)?\z/
+    QUOTED_STRING = /\A"(?:[^"\\\x00-\x1F]|\\[\t\x20-\x7E])*"\z/
+    EMPTY_LIST_MEMBER = /\A[ \t]*\z/
+
+    Metric = Data.define(:name, :duration, :description) do
+      def timed?
+        !duration.nil?
+      end
+    end
+
+    attr_reader :invalid_count
+
+    def initialize(header)
+      @header = header
+      @invalid_count = 0
+      @metrics = parse.freeze
+    end
+
+    def each(&block)
+      @metrics.each(&block)
+    end
+
+    def empty?
+      @metrics.empty?
+    end
+
+    def present?
+      !@header.nil?
+    end
+
+    def size
+      @metrics.size
+    end
+
+    private
+
+    def parse
+      Array(@header).flatten.compact.flat_map do |header|
+        metrics_from(header.to_s)
+      end
+    end
+
+    def metrics_from(header)
+      fields, complete = split(header, ",")
+
+      unless complete
+        @invalid_count += 1
+        fields.pop
+      end
+
+      fields.filter_map do |field|
+        metric_from(field) unless EMPTY_LIST_MEMBER.match?(field)
+      end
+    end
+
+    def metric_from(field)
+      parts, complete = split(field, ";")
+      name = parts.shift.to_s.strip
+
+      if complete && TOKEN.match?(name)
+        metric(name, parameters_from(parts))
+      else
+        @invalid_count += 1
+        nil
+      end
+    end
+
+    def parameters_from(parts)
+      parts.each_with_object({}) do |part, parameters|
+        name, value = part.strip.split("=", 2)
+        next unless name
+
+        name = name.downcase
+        next if parameters.key?(name)
+
+        parameters[name] = parameter_value(value.to_s.strip)
+      end
+    end
+
+    def parameter_value(value)
+      if TOKEN.match?(value)
+        value
+      elsif QUOTED_STRING.match?(value)
+        value[1...-1].gsub(/\\(.)/m, '\1')
+      end
+    end
+
+    def metric(name, parameters)
+      duration = duration_from(parameters["dur"])
+
+      if (parameters.key?("dur") && duration.nil?) ||
+          (parameters.key?("desc") && parameters["desc"].nil?)
+        @invalid_count += 1
+        nil
+      else
+        Metric.new(
+          name: name,
+          duration: duration,
+          description: parameters["desc"]
+        )
+      end
+    end
+
+    def duration_from(value)
+      if value && NUMBER.match?(value)
+        duration = Float(value)
+        duration if duration.finite?
+      end
+    end
+
+    def split(value, delimiter)
+      fields, quoted, escaped = [+""], false, false
+
+      value.each_char do |character|
+        if escaped
+          escaped = false
+        elsif quoted && character == "\\"
+          escaped = true
+        elsif character == "\""
+          quoted = !quoted
+        elsif character == delimiter && !quoted
+          fields << +""
+        end
+
+        fields.last << character unless character == delimiter && !quoted
+      end
+
+      [fields, !quoted && !escaped]
+    end
+  end
+end

data/lib/crawlscope/sitemap.rb

@@ -9,11 +9,12 @@ module Crawlscope
   class Sitemap
     SITEMAP_NAMESPACE = {"xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9"}.freeze
 
-    def initialize(path:, adapter: nil, concurrency: Configuration::DEFAULT_CONCURRENCY, fetch_executor: Configuration::DEFAULT_FETCH_EXECUTOR, timeout_seconds: Configuration::DEFAULT_TIMEOUT_SECONDS)
+    def initialize(path:, adapter: nil, concurrency: Configuration::DEFAULT_CONCURRENCY, fetch_executor: Configuration::DEFAULT_FETCH_EXECUTOR, profile_token: nil, timeout_seconds: Configuration::DEFAULT_TIMEOUT_SECONDS)
       @path = path
       @adapter = adapter
       @concurrency = concurrency
       @fetch_executor = fetch_executor
+      @profile_token = profile_token
       @timeout_seconds = timeout_seconds
     end
 
@@ -34,7 +35,7 @@ module Crawlscope
       end
       return [] if already_visited
 
-      document = Nokogiri::XML(read(source))
+      document = Nokogiri::XML(read(source, base_url: base_url))
       root_name = document.root&.name
       unless %w[sitemapindex urlset].include?(root_name)
         raise ValidationError, "Sitemap #{source} has unexpected root #{root_name.inspect}"
@@ -55,9 +56,16 @@ module Crawlscope
       end
     end
 
-    def read(source)
+    def read(source, base_url:)
       if Url.remote?(source)
-        response = connection.get(source)
+        response = connection.get(source) do |request|
+          RequestHeaders.add_profile_token(
+            request.headers,
+            url: source,
+            base_url: base_url,
+            profile_token: @profile_token
+          )
+        end
         unless response.status.to_i.between?(200, 299)
           raise ValidationError, "Sitemap #{source} returned HTTP #{response.status}"
         end
@@ -92,7 +100,11 @@ module Crawlscope
 
     def connection
       Faraday.new do |faraday|
-        faraday.response :follow_redirects, limit: Http::MAX_REDIRECTS
+        faraday.response(
+          :follow_redirects,
+          limit: Http::MAX_REDIRECTS,
+          callback: RequestHeaders.method(:strip_profile_token_on_cross_origin_redirect)
+        )
         faraday.options.timeout = @timeout_seconds
         faraday.options.open_timeout = @timeout_seconds
         faraday.adapter @adapter if @adapter

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.7.2"
+  VERSION = "0.8.0"
 end

data/lib/generators/crawlscope/templates/initializer.rb.tt

@@ -14,7 +14,7 @@ module CrawlscopeConfiguration
       Crawlscope.configure do |config|
         config.base_url = -> { ENV.fetch("CRAWLSCOPE_BASE_URL", "http://localhost:3000") }
         config.sitemap_path = lambda {
-          ENV.fetch("SITEMAP", Rails.public_path.join("sitemap.xml").to_s)
+          ENV.fetch("SITEMAP", "#{config.base_url.to_s.chomp("/")}/sitemap.xml")
         }
         config.site_name = ENV.fetch("CRAWLSCOPE_SITE_NAME", "Application")
       end

data/test/crawlscope/browser_test.rb

@@ -14,13 +14,14 @@ class CrawlscopeBrowserTest < Minitest::Test
   end
 
   class FakeNetwork
-    attr_reader :cleared, :idle_waits, :status
+    attr_reader :cleared, :idle_waits, :interceptions, :status
 
     def initialize(response:, status: 200)
       @response = response
       @status = status
       @cleared = []
       @idle_waits = []
+      @interceptions = []
     end
 
     def clear(scope)
@@ -29,6 +30,10 @@ class CrawlscopeBrowserTest < Minitest::Test
 
     attr_reader :response
 
+    def intercept(**options)
+      @interceptions << options
+    end
+
     def wait_for_idle(duration:, timeout:)
       @idle_waits << {duration: duration, timeout: timeout}
     end
@@ -43,6 +48,7 @@ class CrawlscopeBrowserTest < Minitest::Test
       @current_url = current_url
       @url = url
       @evaluations = []
+      @callbacks = {}
     end
 
     attr_reader :body
@@ -57,11 +63,40 @@ class CrawlscopeBrowserTest < Minitest::Test
       @visited_url = url
     end
 
+    def on(event, &callback)
+      @callbacks[event] = callback
+    end
+
+    def publish(event, value)
+      @callbacks.fetch(event).call(value)
+    end
+
     attr_reader :url
   end
 
+  class FakeRequest
+    attr_reader :continued_headers, :headers, :url
+
+    def initialize(url:, headers: {})
+      @url = url
+      @headers = headers
+    end
+
+    def continue(headers:)
+      @continued_headers = headers.to_h { |header| [header.fetch(:name), header.fetch(:value)] }
+    end
+  end
+
   def test_fetch_returns_rendered_page
-    network = FakeNetwork.new(response: Response.new(url: "https://example.com/final", headers: {"content-type" => "text/html"}))
+    network = FakeNetwork.new(
+      response: Response.new(
+        url: "https://example.com/final",
+        headers: {
+          "content-type" => "text/html",
+          "server-timing" => "app;dur=47.2"
+        }
+      )
+    )
     page = FakePage.new(network: network, body: "<html><body>Hello</body></html>")
     browser = browser_with(page: page, scroll_page: false)
 
@@ -73,6 +108,7 @@ class CrawlscopeBrowserTest < Minitest::Test
     assert_equal "https://example.com/final", result.normalized_final_url
     assert_equal 200, result.status
     assert result.html?
+    assert_equal 47.2, result.server_timing.first.duration
     assert_equal [], page.evaluations
   end
 
@@ -88,6 +124,21 @@ class CrawlscopeBrowserTest < Minitest::Test
     assert_equal 4, network.idle_waits.size
   end
 
+  def test_profile_token_is_limited_to_same_origin_documents
+    network = FakeNetwork.new(response: nil)
+    page = FakePage.new(network: network)
+    browser_with(page: page, profile_token: "profile-token")
+    same_origin_request = FakeRequest.new(url: "https://example.com/page")
+    cross_origin_request = FakeRequest.new(url: "https://cdn.example.net/frame")
+
+    page.publish(:request, same_origin_request)
+    page.publish(:request, cross_origin_request)
+
+    assert_equal [{resource_type: :document}], network.interceptions
+    assert_equal "profile-token", same_origin_request.continued_headers.fetch("X-Profile-Token")
+    refute_includes cross_origin_request.continued_headers, "X-Profile-Token"
+  end
+
   def test_fetch_falls_back_to_page_url_and_original_url
     page_url_network = FakeNetwork.new(response: nil)
     page_url = FakePage.new(network: page_url_network, url: "https://example.com/page")
@@ -142,14 +193,16 @@ class CrawlscopeBrowserTest < Minitest::Test
 
   private
 
-  def browser_with(page: FakePage.new(network: FakeNetwork.new(response: nil)), browser: FakeBrowser.new, scroll_page: false)
+  def browser_with(page: FakePage.new(network: FakeNetwork.new(response: nil)), browser: FakeBrowser.new, profile_token: nil, scroll_page: false)
     Crawlscope::Browser.allocate.tap do |instance|
       instance.instance_variable_set(:@base_url, "https://example.com")
       instance.instance_variable_set(:@timeout_seconds, 20)
       instance.instance_variable_set(:@network_idle_timeout_seconds, 5)
+      instance.instance_variable_set(:@profile_token, profile_token)
       instance.instance_variable_set(:@scroll_page, scroll_page)
       instance.instance_variable_set(:@browser, browser)
       instance.instance_variable_set(:@page, page)
+      instance.send(:configure_profile_requests)
     end
   end
 end

data/test/crawlscope/configuration_test.rb

@@ -14,6 +14,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
       config.site_name = -> { "Example" }
       config.concurrency = -> { 4 }
       config.fetch_executor = -> { :threaded }
+      config.profile_token = -> { "profile-token" }
     end
 
     audit = Crawlscope.configuration.audit
@@ -22,6 +23,7 @@ class CrawlscopeConfigurationTest < Minitest::Test
     assert_equal "/tmp/sitemap.xml", audit.instance_variable_get(:@sitemap_path)
     assert_equal 4, audit.instance_variable_get(:@concurrency)
     assert_equal :threaded, audit.instance_variable_get(:@fetch_executor)
+    assert_equal "profile-token", audit.instance_variable_get(:@profile_token)
     assert_equal %i[
       indexability
       metadata
@@ -53,17 +55,35 @@ class CrawlscopeConfigurationTest < Minitest::Test
   end
 
   def test_defaults_are_normalized
-    config = Crawlscope::Configuration.new
+    with_profile_token(nil) do
+      config = Crawlscope::Configuration.new
+
+      assert_equal [200, 301, 302], config.allowed_statuses
+      assert_equal 10, config.concurrency
+      assert_equal :async, config.fetch_executor
+      assert_equal 4, config.browser_concurrency
+      assert_equal 5, config.network_idle_timeout_seconds
+      assert_equal :http, config.renderer
+      assert_equal 20, config.timeout_seconds
+      assert_equal $stdout, config.output
+      assert_nil config.profile_token
+      assert config.scroll_page?
+    end
+  end
 
-    assert_equal [200, 301, 302], config.allowed_statuses
-    assert_equal 10, config.concurrency
-    assert_equal :async, config.fetch_executor
-    assert_equal 4, config.browser_concurrency
-    assert_equal 5, config.network_idle_timeout_seconds
-    assert_equal :http, config.renderer
-    assert_equal 20, config.timeout_seconds
-    assert_equal $stdout, config.output
-    assert config.scroll_page?
+  def test_profile_token_defaults_to_the_environment
+    with_profile_token("environment-profile-token") do
+      assert_equal "environment-profile-token", Crawlscope::Configuration.new.profile_token
+    end
+  end
+
+  def test_configured_profile_token_precedes_the_environment
+    with_profile_token("environment-profile-token") do
+      config = Crawlscope::Configuration.new
+      config.profile_token = "configured-profile-token"
+
+      assert_equal "configured-profile-token", config.profile_token
+    end
   end
 
   def test_browser_renderer_defaults_to_threaded_fetch_executor
@@ -119,4 +139,24 @@ class CrawlscopeConfigurationTest < Minitest::Test
 
     assert_equal "Crawlscope concurrency must be an integer >= 1", error.message
   end
+
+  private
+
+  def with_profile_token(value)
+    previous_value = ENV["CRAWLSCOPE_PROFILE_TOKEN"]
+
+    if value.nil?
+      ENV.delete("CRAWLSCOPE_PROFILE_TOKEN")
+    else
+      ENV["CRAWLSCOPE_PROFILE_TOKEN"] = value
+    end
+
+    yield
+  ensure
+    if previous_value.nil?
+      ENV.delete("CRAWLSCOPE_PROFILE_TOKEN")
+    else
+      ENV["CRAWLSCOPE_PROFILE_TOKEN"] = previous_value
+    end
+  end
 end

data/test/crawlscope/crawl_test.rb

@@ -167,6 +167,35 @@ class CrawlscopeCrawlTest < Minitest::Test
     ].sort, result.issues.to_a.map(&:code).uniq.sort
   end
 
+  def test_profile_token_reaches_remote_sitemap_and_page_requests
+    sitemap_request = stub_request(:get, "https://example.com/sitemap.xml")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://example.com/about</loc></url>
+          </urlset>
+        XML
+      )
+    page_request = stub_request(:get, "https://example.com/about")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 200, body: "<html><body>About</body></html>")
+
+    Crawlscope::Crawl.new(
+      base_url: "https://example.com",
+      sitemap_path: "https://example.com/sitemap.xml",
+      rules: [],
+      schema_registry: Crawlscope::SchemaRegistry.default,
+      fetch_executor: :threaded,
+      profile_token: "profile-token"
+    ).call
+
+    assert_requested sitemap_request
+    assert_requested page_request
+  end
+
   def test_uses_browser_when_renderer_is_browser
     File.write(
       @sitemap_path,

data/test/crawlscope/http_test.rb

@@ -5,13 +5,24 @@ require "test_helper"
 class CrawlscopeHttpTest < Minitest::Test
   def test_fetch_parses_html_response
     stub_request(:get, "https://example.com/page")
-      .to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<html><body>Hello</body></html>")
+      .to_return(
+        status: 200,
+        headers: {
+          "Content-Type" => "text/html",
+          "Server-Timing" => 'db;dur=12.5;desc="Primary database"'
+        },
+        body: "<html><body>Hello</body></html>"
+      )
 
     page = Crawlscope::Http.new(base_url: "https://example.com", timeout_seconds: 2).fetch("https://example.com/page")
 
     assert_equal 200, page.status
     assert page.html?
     assert_equal "Hello", page.doc.at_css("body").text
+    timing = page.server_timing
+    assert_same timing, page.server_timing
+    assert_equal 12.5, timing.first.duration
+    assert_equal "Primary database", timing.first.description
   end
 
   def test_fetch_parses_responses_without_content_type_as_html
@@ -23,6 +34,54 @@ class CrawlscopeHttpTest < Minitest::Test
     assert page.html?
   end
 
+  def test_fetch_sends_profile_token_to_same_origin_requests
+    request = stub_request(:get, "https://example.com/page")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 200, body: "<html></html>")
+
+    Crawlscope::Http.new(
+      base_url: "https://example.com",
+      timeout_seconds: 2,
+      profile_token: "profile-token"
+    ).fetch("https://example.com/page")
+
+    assert_requested request
+  end
+
+  def test_fetch_preserves_profile_token_on_same_origin_redirects
+    stub_request(:get, "https://example.com/start")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 302, headers: {"Location" => "/final"})
+    final_request = stub_request(:get, "https://example.com/final")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 200, body: "<html></html>")
+
+    Crawlscope::Http.new(
+      base_url: "https://example.com",
+      timeout_seconds: 2,
+      profile_token: "profile-token"
+    ).fetch("https://example.com/start")
+
+    assert_requested final_request
+  end
+
+  def test_fetch_removes_profile_token_from_cross_origin_redirects
+    stub_request(:get, "https://example.com/start")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(status: 302, headers: {"Location" => "https://other.example/final"})
+    final_request = stub_request(:get, "https://other.example/final")
+      .with { |request| !request.headers.key?("X-Profile-Token") }
+      .to_return(status: 200, body: "<html></html>")
+
+    Crawlscope::Http.new(
+      base_url: "https://example.com",
+      timeout_seconds: 2,
+      profile_token: "profile-token"
+    ).fetch("https://example.com/start")
+
+    assert_requested final_request
+  end
+
   def test_fetch_leaves_non_html_response_unparsed
     stub_request(:get, "https://example.com/feed.xml")
       .to_return(status: 200, headers: {"content-type" => "application/xml"}, body: "<feed></feed>")

data/test/crawlscope/install_generator_test.rb

@@ -16,11 +16,14 @@ class CrawlscopeInstallGeneratorTest < Minitest::Test
       Crawlscope::Generators::InstallGenerator.start([], destination_root: destination)
 
       initializer = File.join(destination, "config/initializers/crawlscope.rb")
+      initializer_contents = File.read(initializer)
       rakefile = File.read(File.join(destination, "Rakefile"))
 
       assert File.exist?(initializer)
-      assert_includes File.read(initializer), "module CrawlscopeConfiguration"
-      assert_includes File.read(initializer), "CrawlscopeConfiguration.apply if defined?(Crawlscope)"
+      assert_includes initializer_contents, "module CrawlscopeConfiguration"
+      assert_includes initializer_contents, "CrawlscopeConfiguration.apply if defined?(Crawlscope)"
+      assert_includes initializer_contents, "\#{config.base_url.to_s.chomp(\"/\")}/sitemap.xml"
+      refute_includes initializer_contents, "Rails.public_path"
       assert_includes rakefile, 'require "crawlscope/tasks"'
       assert_operator rakefile.index('require "crawlscope/tasks"'), :<, rakefile.index('require_relative "config/application"')
 
@@ -28,14 +31,6 @@ class CrawlscopeInstallGeneratorTest < Minitest::Test
       assert status.success?, stderr
 
       script = <<~RUBY
-        require "pathname"
-
-        module Rails
-          def self.public_path
-            Pathname.new("public")
-          end
-        end
-
         require "crawlscope"
         load ARGV.fetch(0)
 
@@ -43,7 +38,10 @@ class CrawlscopeInstallGeneratorTest < Minitest::Test
         CrawlscopeConfiguration.apply
         abort "configuration replaced" unless Crawlscope.configuration.equal?(configuration)
         abort "base URL missing" unless configuration.base_url == "http://localhost:3000"
-        abort "sitemap missing" unless configuration.sitemap_path == "public/sitemap.xml"
+        abort "sitemap missing" unless configuration.sitemap_path == "http://localhost:3000/sitemap.xml"
+
+        ENV["CRAWLSCOPE_BASE_URL"] = "https://example.com"
+        abort "configured sitemap missing" unless configuration.sitemap_path == "https://example.com/sitemap.xml"
       RUBY
       _stdout, stderr, status = Open3.capture3(
         RbConfig.ruby,