crawlscope 0.7.2 → 0.8.1

data/lib/crawlscope/server_timing/reporter.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Crawlscope
+  class ServerTiming
+    class Reporter
+      SLOW_PAGE_THRESHOLD_MS = 50
+
+      def initialize(io:)
+        @io = io
+      end
+
+      def report(summary, base_url:)
+        return unless summary.present?
+
+        @io.puts("")
+        @io.puts("Server Timing:")
+        @io.puts(
+          "  Coverage: #{summary.coverage}/#{summary.pages.size} pages " \
+          "(#{percentage(summary.coverage, summary.pages.size)}); " \
+          "durations on #{summary.duration_pages} #{pluralize(:page, summary.duration_pages)}"
+        )
+
+        report_metrics(summary.metrics)
+        report_signals(summary.signals)
+        report_worst_pages(summary.worst_pages(limit: summary.pages.size), base_url: base_url)
+        @io.puts("  Ignored malformed entries: #{summary.invalid_count}") if summary.invalid_count.positive?
+      end
+
+      private
+
+      def report_metrics(metrics)
+        return if metrics.empty?
+
+        @io.puts("  Response times (ms):")
+        metrics.each do |metric|
+          label = metric[:name]
+          label += " #{description(metric[:description])}" if metric[:description]
+
+          @io.puts(
+            "    #{label}: " \
+            "#{metric[:samples]} #{pluralize(:sample, metric[:samples])} / " \
+            "#{metric[:pages]} #{pluralize(:page, metric[:pages])}; " \
+            "avg #{duration(metric[:average])}; " \
+            "p50 #{duration(metric[:p50])}; " \
+            "p95 #{duration(metric[:p95])}; " \
+            "max #{duration(metric[:maximum])}"
+          )
+        end
+      end
+
+      def report_signals(signals)
+        return if signals.empty?
+
+        @io.puts("  Signals:")
+        signals.each do |signal|
+          label = signal[:name]
+          label += " #{description(signal[:description])}" if signal[:description]
+
+          @io.puts(
+            "    #{label}: " \
+            "#{signal[:samples]} #{pluralize(:sample, signal[:samples])} / " \
+            "#{signal[:pages]} #{pluralize(:page, signal[:pages])}"
+          )
+        end
+      end
+
+      def report_worst_pages(samples, base_url:)
+        samples = samples.select { |sample| sample[:metric].duration > SLOW_PAGE_THRESHOLD_MS }
+        return if samples.empty?
+
+        @io.puts("  Pages above #{SLOW_PAGE_THRESHOLD_MS} ms:")
+        samples.each do |sample|
+          metric = sample[:metric]
+          detail = metric.description ? " #{description(metric.description)}" : ""
+          @io.puts(
+            "    #{relative_url(sample[:page].url, base_url: base_url)}: " \
+            "#{duration(metric.duration)} (#{metric.name}#{detail})"
+          )
+        end
+      end
+
+      def duration(value)
+        "#{format("%.3f", value).sub(/\.?0+\z/, "")}ms"
+      end
+
+      def description(value)
+        value = value.to_s
+        value.inspect
+      end
+
+      def percentage(numerator, denominator)
+        return "0.0%" if denominator.zero?
+
+        format("%.1f%%", numerator.fdiv(denominator) * 100)
+      end
+
+      def relative_url(url, base_url:)
+        uri = URI.parse(url)
+        base_uri = URI.parse(base_url)
+        return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
+
+        uri.request_uri
+      rescue URI::InvalidURIError
+        url
+      end
+
+      def pluralize(word, count)
+        (count == 1) ? word.to_s : "#{word}s"
+      end
+    end
+  end
+end

data/lib/crawlscope/server_timing/summary.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class ServerTiming
+    class Summary
+      WORST_PAGE_LIMIT = 10
+
+      attr_reader :pages
+
+      def initialize(pages)
+        @pages = pages
+        @timings = pages.filter_map do |page|
+          timing = page.server_timing
+          [page, timing] if timing.present?
+        end
+      end
+
+      def present?
+        @timings.any?
+      end
+
+      def duration_pages
+        @timings.count { |_page, timing| timing.any?(&:timed?) }
+      end
+
+      def invalid_count
+        @timings.sum { |_page, timing| timing.invalid_count }
+      end
+
+      def metrics
+        timed_metrics
+          .group_by { |_page, metric| [metric.name, metric.description] }
+          .map { |(name, description), samples| metric_summary(name, description, samples) }
+          .sort_by { |metric| [-metric[:p95], -metric[:maximum], metric[:name]] }
+      end
+
+      def signals
+        all_metrics
+          .reject { |_page, metric| metric.timed? }
+          .group_by { |_page, metric| [metric.name, metric.description] }
+          .map { |(name, description), samples| signal_summary(name, description, samples) }
+          .sort_by { |signal| [-signal[:samples], signal[:name], signal[:description].to_s] }
+      end
+
+      def worst_pages(limit: WORST_PAGE_LIMIT)
+        @timings
+          .filter_map { |page, timing| worst_metric(page, timing) }
+          .min_by(limit) { |sample| [-sample[:metric].duration, sample[:page].url] }
+          .sort_by { |sample| [-sample[:metric].duration, sample[:page].url] }
+      end
+
+      def coverage
+        @timings.size
+      end
+
+      private
+
+      def timed_metrics
+        all_metrics.select { |_page, metric| metric.timed? }
+      end
+
+      def all_metrics
+        @all_metrics ||= @timings.flat_map do |page, timing|
+          timing.map { |metric| [page, metric] }
+        end
+      end
+
+      def metric_summary(name, description, samples)
+        durations = samples.map { |_page, metric| metric.duration }.sort
+
+        {
+          name: name,
+          description: description,
+          samples: samples.size,
+          pages: samples.map(&:first).uniq.size,
+          average: durations.sum.fdiv(durations.size),
+          p50: percentile(durations, 0.50),
+          p95: percentile(durations, 0.95),
+          maximum: durations.last
+        }
+      end
+
+      def signal_summary(name, description, samples)
+        {
+          name: name,
+          description: description,
+          samples: samples.size,
+          pages: samples.map(&:first).uniq.size
+        }
+      end
+
+      def percentile(values, percentile)
+        position = (values.size - 1) * percentile
+        lower = values[position.floor]
+        upper = values[position.ceil]
+
+        lower + (upper - lower) * (position - position.floor)
+      end
+
+      def worst_metric(page, timing)
+        if (metric = timing.select(&:timed?).max_by(&:duration))
+          {page: page, metric: metric}
+        end
+      end
+    end
+  end
+end

data/lib/crawlscope/server_timing.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  class ServerTiming
+    include Enumerable
+
+    TOKEN = /\A[!#$%&'*+\-.^_`|~0-9A-Za-z]+\z/
+    NUMBER = /\A-?(?:(?:\d+(?:\.\d*)?)|(?:\.\d+))(?:[eE][+-]?\d+)?\z/
+    QUOTED_STRING = /\A"(?:[^"\\\x00-\x1F]|\\[\t\x20-\x7E])*"\z/
+    EMPTY_LIST_MEMBER = /\A[ \t]*\z/
+
+    Metric = Data.define(:name, :duration, :description) do
+      def timed?
+        !duration.nil?
+      end
+    end
+
+    attr_reader :invalid_count
+
+    def initialize(header)
+      @header = header
+      @invalid_count = 0
+      @metrics = parse.freeze
+    end
+
+    def each(&block)
+      @metrics.each(&block)
+    end
+
+    def empty?
+      @metrics.empty?
+    end
+
+    def present?
+      !@header.nil?
+    end
+
+    def size
+      @metrics.size
+    end
+
+    private
+
+    def parse
+      Array(@header).flatten.compact.flat_map do |header|
+        metrics_from(header.to_s)
+      end
+    end
+
+    def metrics_from(header)
+      fields, complete = split(header, ",")
+
+      unless complete
+        @invalid_count += 1
+        fields.pop
+      end
+
+      fields.filter_map do |field|
+        metric_from(field) unless EMPTY_LIST_MEMBER.match?(field)
+      end
+    end
+
+    def metric_from(field)
+      parts, complete = split(field, ";")
+      name = parts.shift.to_s.strip
+
+      if complete && TOKEN.match?(name)
+        metric(name, parameters_from(parts))
+      else
+        @invalid_count += 1
+        nil
+      end
+    end
+
+    def parameters_from(parts)
+      parts.each_with_object({}) do |part, parameters|
+        name, value = part.strip.split("=", 2)
+        next unless name
+
+        name = name.downcase
+        next if parameters.key?(name)
+
+        parameters[name] = parameter_value(value.to_s.strip)
+      end
+    end
+
+    def parameter_value(value)
+      if TOKEN.match?(value)
+        value
+      elsif QUOTED_STRING.match?(value)
+        value[1...-1].gsub(/\\(.)/m, '\1')
+      end
+    end
+
+    def metric(name, parameters)
+      duration = duration_from(parameters["dur"])
+
+      if (parameters.key?("dur") && duration.nil?) ||
+          (parameters.key?("desc") && parameters["desc"].nil?)
+        @invalid_count += 1
+        nil
+      else
+        Metric.new(
+          name: name,
+          duration: duration,
+          description: parameters["desc"]
+        )
+      end
+    end
+
+    def duration_from(value)
+      if value && NUMBER.match?(value)
+        duration = Float(value)
+        duration if duration.finite?
+      end
+    end
+
+    def split(value, delimiter)
+      fields, quoted, escaped = [+""], false, false
+
+      value.each_char do |character|
+        if escaped
+          escaped = false
+        elsif quoted && character == "\\"
+          escaped = true
+        elsif character == "\""
+          quoted = !quoted
+        elsif character == delimiter && !quoted
+          fields << +""
+        end
+
+        fields.last << character unless character == delimiter && !quoted
+      end
+
+      [fields, !quoted && !escaped]
+    end
+  end
+end

data/lib/crawlscope/sitemap.rb

@@ -9,11 +9,12 @@ module Crawlscope
   class Sitemap
     SITEMAP_NAMESPACE = {"xmlns" => "http://www.sitemaps.org/schemas/sitemap/0.9"}.freeze
 
-    def initialize(path:, adapter: nil, concurrency: Configuration::DEFAULT_CONCURRENCY, fetch_executor: Configuration::DEFAULT_FETCH_EXECUTOR, timeout_seconds: Configuration::DEFAULT_TIMEOUT_SECONDS)
+    def initialize(path:, adapter: nil, concurrency: Configuration::DEFAULT_CONCURRENCY, fetch_executor: Configuration::DEFAULT_FETCH_EXECUTOR, profile_token: nil, timeout_seconds: Configuration::DEFAULT_TIMEOUT_SECONDS)
       @path = path
       @adapter = adapter
       @concurrency = concurrency
       @fetch_executor = fetch_executor
+      @profile_token = profile_token
       @timeout_seconds = timeout_seconds
     end
 
@@ -34,7 +35,7 @@ module Crawlscope
       end
       return [] if already_visited
 
-      document = Nokogiri::XML(read(source))
+      document = Nokogiri::XML(read(source, base_url: base_url))
       root_name = document.root&.name
       unless %w[sitemapindex urlset].include?(root_name)
         raise ValidationError, "Sitemap #{source} has unexpected root #{root_name.inspect}"
@@ -55,9 +56,16 @@ module Crawlscope
       end
     end
 
-    def read(source)
+    def read(source, base_url:)
       if Url.remote?(source)
-        response = connection.get(source)
+        response = connection.get(source) do |request|
+          RequestHeaders.add_profile_token(
+            request.headers,
+            url: source,
+            base_url: base_url,
+            profile_token: @profile_token
+          )
+        end
         unless response.status.to_i.between?(200, 299)
           raise ValidationError, "Sitemap #{source} returned HTTP #{response.status}"
         end
@@ -92,7 +100,11 @@ module Crawlscope
 
     def connection
       Faraday.new do |faraday|
-        faraday.response :follow_redirects, limit: Http::MAX_REDIRECTS
+        faraday.response(
+          :follow_redirects,
+          limit: Http::MAX_REDIRECTS,
+          callback: RequestHeaders.method(:strip_profile_token_on_cross_origin_redirect)
+        )
         faraday.options.timeout = @timeout_seconds
         faraday.options.open_timeout = @timeout_seconds
         faraday.adapter @adapter if @adapter

data/lib/crawlscope/structured_data/error_formatter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Crawlscope
+  module StructuredData
+    class ErrorFormatter
+      def self.format(error)
+        field = (error[:field] || error["field"]).to_s.sub(/\A#\/?/, "")
+        type = error[:type] || error["type"]
+        issue = error[:issue] || error["issue"] || "Unknown error"
+
+        if (match = issue.match(/did not contain a required property of '([^']+)'/))
+          field = [field, match[1]].reject(&:empty?).join("/")
+          message = "is required"
+        else
+          message = concise_message(issue, error[:length] || error["length"])
+        end
+
+        location = [type, field.empty? ? nil : field].compact.join(".")
+        "#{location.empty? ? "$" : location}: #{message}"
+      end
+
+      def self.concise_message(issue, length)
+        if length
+          unit = (length == 1) ? "character" : "characters"
+          measured_length = "#{length} #{unit}"
+        end
+        case issue
+        when /was not of a maximum string length of (\d+)/
+          length ? "#{measured_length}; maximum #{Regexp.last_match(1)}" : "exceeds #{Regexp.last_match(1)} characters"
+        when /was not of a minimum string length of (\d+)/
+          length ? "#{measured_length}; minimum #{Regexp.last_match(1)}" : "must have at least #{Regexp.last_match(1)} characters"
+        when /of type (\w+) did not match the following type: (\w+)/
+          "must be #{Regexp.last_match(2)} (got #{Regexp.last_match(1)})"
+        else
+          issue.sub(/\AThe property '[^']*' /, "")
+            .sub(/ in schema [\h-]{36}(?:#\S*)?\z/, "")
+        end
+      end
+      private_class_method :concise_message
+    end
+  end
+end

data/lib/crawlscope/structured_data/reporter.rb

@@ -67,10 +67,9 @@ module Crawlscope
           @io.puts("• #{extract_path(entry.url)}")
 
           entry.errors.each do |error|
+            @io.puts("    #{source_label(error[:source])}:")
             error[:errors].each do |validation_error|
-              field = validation_error[:field] || validation_error["field"] || "$"
-              issue = validation_error[:issue] || validation_error["issue"] || "Unknown error"
-              @io.puts("    - #{field}: #{issue}")
+              @io.puts("      - #{ErrorFormatter.format(validation_error)}")
             end
           end
         end
@@ -83,9 +82,7 @@ module Crawlscope
       end
 
       def report_entry(entry, debug:)
-        @io.puts("=" * 80)
         @io.puts("URL: #{entry.url}")
-        @io.puts("=" * 80)
 
         if entry.fetch_error
           @io.puts("Error: #{entry.fetch_error}")
@@ -124,19 +121,22 @@ module Crawlscope
 
       def report_validation(entry)
         @io.puts("")
-        @io.puts("Validation results:")
-
         if entry.errors.empty?
-          @io.puts("  All valid!")
+          @io.puts("Validation: OK")
         else
+          @io.puts("Validation: FAILED")
           entry.errors.each do |error|
-            @io.puts("  #{error[:type]}: INVALID [#{error[:source]}]")
+            @io.puts("  #{source_label(error[:source])}:")
             error[:errors].each do |validation_error|
-              @io.puts("    - field: #{validation_error[:field]}, issue: #{validation_error[:issue]}")
+              @io.puts("    - #{ErrorFormatter.format(validation_error)}")
             end
           end
         end
       end
+
+      def source_label(source)
+        (source == "json-ld") ? "JSON-LD" : source
+      end
     end
   end
 end

data/lib/crawlscope/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Crawlscope
-  VERSION = "0.7.2"
+  VERSION = "0.8.1"
 end

data/lib/generators/crawlscope/templates/initializer.rb.tt

@@ -14,7 +14,7 @@ module CrawlscopeConfiguration
       Crawlscope.configure do |config|
         config.base_url = -> { ENV.fetch("CRAWLSCOPE_BASE_URL", "http://localhost:3000") }
         config.sitemap_path = lambda {
-          ENV.fetch("SITEMAP", Rails.public_path.join("sitemap.xml").to_s)
+          ENV.fetch("SITEMAP", "#{config.base_url.to_s.chomp("/")}/sitemap.xml")
         }
         config.site_name = ENV.fetch("CRAWLSCOPE_SITE_NAME", "Application")
       end

data/test/crawlscope/browser_test.rb

@@ -14,13 +14,14 @@ class CrawlscopeBrowserTest < Minitest::Test
   end
 
   class FakeNetwork
-    attr_reader :cleared, :idle_waits, :status
+    attr_reader :cleared, :idle_waits, :interceptions, :status
 
     def initialize(response:, status: 200)
       @response = response
       @status = status
       @cleared = []
       @idle_waits = []
+      @interceptions = []
     end
 
     def clear(scope)
@@ -29,6 +30,10 @@ class CrawlscopeBrowserTest < Minitest::Test
 
     attr_reader :response
 
+    def intercept(**options)
+      @interceptions << options
+    end
+
     def wait_for_idle(duration:, timeout:)
       @idle_waits << {duration: duration, timeout: timeout}
     end
@@ -43,6 +48,7 @@ class CrawlscopeBrowserTest < Minitest::Test
       @current_url = current_url
       @url = url
       @evaluations = []
+      @callbacks = {}
     end
 
     attr_reader :body
@@ -57,11 +63,40 @@ class CrawlscopeBrowserTest < Minitest::Test
       @visited_url = url
     end
 
+    def on(event, &callback)
+      @callbacks[event] = callback
+    end
+
+    def publish(event, value)
+      @callbacks.fetch(event).call(value)
+    end
+
     attr_reader :url
   end
 
+  class FakeRequest
+    attr_reader :continued_headers, :headers, :url
+
+    def initialize(url:, headers: {})
+      @url = url
+      @headers = headers
+    end
+
+    def continue(headers:)
+      @continued_headers = headers.to_h { |header| [header.fetch(:name), header.fetch(:value)] }
+    end
+  end
+
   def test_fetch_returns_rendered_page
-    network = FakeNetwork.new(response: Response.new(url: "https://example.com/final", headers: {"content-type" => "text/html"}))
+    network = FakeNetwork.new(
+      response: Response.new(
+        url: "https://example.com/final",
+        headers: {
+          "content-type" => "text/html",
+          "server-timing" => "app;dur=47.2"
+        }
+      )
+    )
     page = FakePage.new(network: network, body: "<html><body>Hello</body></html>")
     browser = browser_with(page: page, scroll_page: false)
 
@@ -73,6 +108,7 @@ class CrawlscopeBrowserTest < Minitest::Test
     assert_equal "https://example.com/final", result.normalized_final_url
     assert_equal 200, result.status
     assert result.html?
+    assert_equal 47.2, result.server_timing.first.duration
     assert_equal [], page.evaluations
   end
 
@@ -88,6 +124,21 @@ class CrawlscopeBrowserTest < Minitest::Test
     assert_equal 4, network.idle_waits.size
   end
 
+  def test_profile_token_is_limited_to_same_origin_documents
+    network = FakeNetwork.new(response: nil)
+    page = FakePage.new(network: network)
+    browser_with(page: page, profile_token: "profile-token")
+    same_origin_request = FakeRequest.new(url: "https://example.com/page")
+    cross_origin_request = FakeRequest.new(url: "https://cdn.example.net/frame")
+
+    page.publish(:request, same_origin_request)
+    page.publish(:request, cross_origin_request)
+
+    assert_equal [{resource_type: :document}], network.interceptions
+    assert_equal "profile-token", same_origin_request.continued_headers.fetch("X-Profile-Token")
+    refute_includes cross_origin_request.continued_headers, "X-Profile-Token"
+  end
+
   def test_fetch_falls_back_to_page_url_and_original_url
     page_url_network = FakeNetwork.new(response: nil)
     page_url = FakePage.new(network: page_url_network, url: "https://example.com/page")
@@ -142,14 +193,16 @@ class CrawlscopeBrowserTest < Minitest::Test
 
   private
 
-  def browser_with(page: FakePage.new(network: FakeNetwork.new(response: nil)), browser: FakeBrowser.new, scroll_page: false)
+  def browser_with(page: FakePage.new(network: FakeNetwork.new(response: nil)), browser: FakeBrowser.new, profile_token: nil, scroll_page: false)
     Crawlscope::Browser.allocate.tap do |instance|
       instance.instance_variable_set(:@base_url, "https://example.com")
       instance.instance_variable_set(:@timeout_seconds, 20)
       instance.instance_variable_set(:@network_idle_timeout_seconds, 5)
+      instance.instance_variable_set(:@profile_token, profile_token)
       instance.instance_variable_set(:@scroll_page, scroll_page)
       instance.instance_variable_set(:@browser, browser)
       instance.instance_variable_set(:@page, page)
+      instance.send(:configure_profile_requests)
     end
   end
 end