crawlscope 0.7.2 → 0.8.0

data/test/crawlscope/reporter_test.rb

@@ -10,7 +10,7 @@ class CrawlscopeReporterTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
       urls: ["https://example.com"],
-      pages: [Object.new],
+      pages: [page],
       issues: Crawlscope::IssueCollection.new
     )
 
@@ -21,6 +21,7 @@ class CrawlscopeReporterTest < Minitest::Test
     assert_includes output, "Crawlscope validation"
     assert_includes output, "Status: OK"
     refute_includes output, "Status: FAILED"
+    refute_includes output, "Server Timing:"
   end
 
   def test_reports_warning_result_with_grouped_one_line_issues
@@ -46,7 +47,7 @@ class CrawlscopeReporterTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
       urls: ["https://example.com/a", "https://example.com/b"],
-      pages: [Object.new, Object.new],
+      pages: [page("/one"), page("/two")],
       issues: issues
     )
 
@@ -77,7 +78,7 @@ class CrawlscopeReporterTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
       urls: ["https://example.com/a"],
-      pages: [Object.new],
+      pages: [page],
       issues: issues
     )
 
@@ -107,7 +108,7 @@ class CrawlscopeReporterTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
       urls: ["https://example.com"],
-      pages: [Object.new],
+      pages: [page],
       issues: issues
     )
 
@@ -137,7 +138,7 @@ class CrawlscopeReporterTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
       urls: ["https://example.com/a"],
-      pages: [Object.new],
+      pages: [page],
       issues: issues
     )
 
@@ -164,7 +165,7 @@ class CrawlscopeReporterTest < Minitest::Test
       base_url: "https://example.com",
       sitemap_path: "/tmp/sitemap.xml",
       urls: ["https://example.com"],
-      pages: [Object.new],
+      pages: [page],
       issues: issues
     )
 
@@ -176,4 +177,61 @@ class CrawlscopeReporterTest < Minitest::Test
     assert_includes output, "  - /overview-1  indexable internal page is missing from sitemap  source: /source-1"
     assert_includes output, "  - /overview-4  indexable internal page is missing from sitemap  source: /source-4"
   end
+
+  def test_reports_server_timing_coverage_percentiles_signals_and_worst_pages
+    io = StringIO.new
+    result = Crawlscope::Result.new(
+      base_url: "https://example.com",
+      sitemap_path: "/tmp/sitemap.xml",
+      urls: [
+        "https://example.com/fast",
+        "https://example.com/slow",
+        "https://example.com/malformed"
+      ],
+      pages: [
+        page(
+          "/fast",
+          server_timing: 'total;dur=100, db;dur=20;desc="Primary", cache;desc="HIT"'
+        ),
+        page(
+          "/slow",
+          server_timing: 'total;dur=300, db;dur=80, cache;desc="MISS"'
+        ),
+        page("/malformed", server_timing: "bad metric;dur=10")
+      ],
+      issues: Crawlscope::IssueCollection.new
+    )
+
+    Crawlscope::Reporter.new(io: io).report(result)
+
+    output = io.string
+    assert_includes output, "Server Timing:"
+    assert_includes output, "Coverage: 3/3 pages (100.0%); durations on 2 pages"
+    assert_includes output, "Duration metrics (dur; milliseconds assumed):"
+    assert_includes output, "total: 2 samples / 2 pages; avg 200ms; p50 200ms; p95 290ms; max 300ms"
+    assert_includes output, 'db "Primary": 1 sample / 1 page'
+    assert_includes output, 'cache "HIT": 1 sample / 1 page'
+    assert_includes output, "Worst pages:"
+    assert_includes output, "/slow: 300ms (total)"
+    assert_includes output, "Ignored malformed entries: 1"
+  end
+
+  private
+
+  def page(path = "/", server_timing: nil)
+    url = "https://example.com#{path}"
+    headers = {}
+    headers["Server-Timing"] = server_timing unless server_timing.nil?
+
+    Crawlscope::Page.new(
+      url: url,
+      normalized_url: url,
+      final_url: url,
+      normalized_final_url: url,
+      status: 200,
+      headers: headers,
+      body: "",
+      doc: nil
+    )
+  end
 end

data/test/crawlscope/run_test.rb

@@ -130,29 +130,21 @@ class CrawlscopeRunTest < Minitest::Test
     )
   end
 
-  def test_validate_prefers_local_sitemap_for_localhost
+  def test_validate_uses_http_sitemap_for_localhost
     result = FakeResult.new(reported: true)
     configuration = FakeConfiguration.new(result: result, base_url: "http://localhost:3000", sitemap_path: nil)
     reporter = FakeReporter.new
-    tmp_dir = Dir.mktmpdir
-    sitemap_path = File.join(tmp_dir, "public", "sitemap.xml")
-    FileUtils.mkdir_p(File.dirname(sitemap_path))
-    File.write(sitemap_path, "<urlset></urlset>")
 
-    Dir.chdir(tmp_dir) do
-      Crawlscope::Run.new(configuration: configuration, reporter: reporter).validate
-    end
+    Crawlscope::Run.new(configuration: configuration, reporter: reporter).validate
 
     assert_equal(
       {
         base_url: "http://localhost:3000",
-        sitemap_path: sitemap_path,
+        sitemap_path: "http://localhost:3000/sitemap.xml",
         rule_names: nil
       },
       configuration.received_arguments
     )
-  ensure
-    FileUtils.rm_rf(tmp_dir) if tmp_dir
   end
 
   def test_validate_json_ld_reports_valid_structured_data

data/test/crawlscope/server_timing_summary_test.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeServerTimingSummaryTest < Minitest::Test
+  def test_aggregates_coverage_percentiles_signals_and_worst_pages
+    pages = [
+      page("/fast", 'total;dur=100, db;dur=20, cache;desc="HIT"'),
+      page("/typical", 'total;dur=200, db;dur=50, cache;desc="MISS"'),
+      page("/slow", 'total;dur=300, db;dur=80, cache;desc="HIT", miss'),
+      page("/no-header"),
+      page("/malformed", "bad metric;dur=10")
+    ]
+
+    summary = Crawlscope::ServerTiming::Summary.new(pages)
+
+    assert summary.present?
+    assert_equal 5, summary.pages.size
+    assert_equal 4, summary.coverage
+    assert_equal 3, summary.duration_pages
+    assert_equal 1, summary.invalid_count
+
+    total = summary.metrics.find { |metric| metric[:name] == "total" }
+    assert_equal 3, total[:samples]
+    assert_equal 3, total[:pages]
+    assert_in_delta 200.0, total[:average]
+    assert_in_delta 200.0, total[:p50]
+    assert_in_delta 290.0, total[:p95]
+    assert_in_delta 300.0, total[:maximum]
+
+    hit = summary.signals.find do |signal|
+      signal[:name] == "cache" && signal[:description] == "HIT"
+    end
+    assert_equal 2, hit[:samples]
+    assert_equal 2, hit[:pages]
+
+    assert_equal(
+      [
+        ["https://example.com/slow", "total", 300.0],
+        ["https://example.com/typical", "total", 200.0]
+      ],
+      summary.worst_pages(limit: 2).map do |sample|
+        [sample[:page].url, sample[:metric].name, sample[:metric].duration]
+      end
+    )
+  end
+
+  def test_is_absent_when_no_pages_publish_the_header
+    summary = Crawlscope::ServerTiming::Summary.new([page("/one"), page("/two")])
+
+    refute summary.present?
+    assert_empty summary.metrics
+    assert_empty summary.signals
+    assert_empty summary.worst_pages
+  end
+
+  def test_returns_ten_worst_pages_by_default
+    pages = 12.times.map do |index|
+      page("/page-#{index + 1}", "total;dur=#{index + 1}")
+    end
+
+    summary = Crawlscope::ServerTiming::Summary.new(pages)
+
+    assert_equal 10, summary.worst_pages.size
+    assert_equal "https://example.com/page-12", summary.worst_pages.first[:page].url
+    assert_equal "https://example.com/page-3", summary.worst_pages.last[:page].url
+  end
+
+  private
+
+  def page(path, server_timing = nil)
+    url = "https://example.com#{path}"
+    headers = {}
+    headers["Server-Timing"] = server_timing unless server_timing.nil?
+
+    Crawlscope::Page.new(
+      url: url,
+      normalized_url: url,
+      final_url: url,
+      normalized_final_url: url,
+      status: 200,
+      headers: headers,
+      body: "",
+      doc: nil
+    )
+  end
+end

data/test/crawlscope/server_timing_test.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeServerTimingTest < Minitest::Test
+  def test_parses_metrics_durations_descriptions_and_unknown_parameters
+    timing = Crawlscope::ServerTiming.new(
+      'miss, db;dur=53, cache;desc="Cache Read";dur=23.2;region=atl'
+    )
+
+    assert timing.present?
+    assert_equal 0, timing.invalid_count
+    assert_equal %w[miss db cache], timing.map(&:name)
+
+    cache = timing.to_a.last
+    assert_equal 23.2, cache.duration
+    assert_equal "Cache Read", cache.description
+  end
+
+  def test_preserves_duplicate_metrics_and_uses_the_first_duplicate_parameter
+    timing = Crawlscope::ServerTiming.new(
+      ["db;DUR=10;dur=20", "db;dur=30"]
+    )
+
+    assert_equal [10.0, 30.0], timing.map(&:duration)
+    assert_equal 0, timing.invalid_count
+  end
+
+  def test_parses_rails_notification_names
+    timing = Crawlscope::ServerTiming.new(
+      "sql.active_record;dur=82.4, " \
+      "render_template.action_view;dur=10.2, " \
+      "process_action.action_controller;dur=101.5"
+    )
+
+    assert_equal(
+      %w[
+        sql.active_record
+        render_template.action_view
+        process_action.action_controller
+      ],
+      timing.map(&:name)
+    )
+    assert_equal 0, timing.invalid_count
+  end
+
+  def test_handles_delimiters_and_escapes_inside_quoted_descriptions
+    timing = Crawlscope::ServerTiming.new(
+      'cache;desc="Cache, read; \\"fast\\"";dur=23.2'
+    )
+
+    assert_equal 1, timing.size
+    assert_equal 'Cache, read; "fast"', timing.first.description
+    assert_equal 0, timing.invalid_count
+  end
+
+  def test_reports_invalid_entries_without_discarding_valid_metrics
+    timing = Crawlscope::ServerTiming.new(
+      "bad metric;dur=2, db;dur=nope, app;dur=3, cache;broken"
+    )
+
+    assert_equal %w[app cache], timing.map(&:name)
+    assert_equal 3.0, timing.first.duration
+    assert_equal 2, timing.invalid_count
+  end
+
+  def test_rejects_malformed_standard_parameters
+    timing = Crawlscope::ServerTiming.new(
+      "db;dur, cache;desc=not quoted, app;dur=3"
+    )
+
+    assert_equal ["app"], timing.map(&:name)
+    assert_equal 2, timing.invalid_count
+  end
+
+  def test_keeps_complete_metrics_before_an_unterminated_description
+    timing = Crawlscope::ServerTiming.new(
+      'db;dur=5, cache;desc="unterminated'
+    )
+
+    assert_equal ["db"], timing.map(&:name)
+    assert_equal 1, timing.invalid_count
+  end
+
+  def test_ignores_empty_list_members
+    timing = Crawlscope::ServerTiming.new(", db;dur=5, , app;dur=3,")
+
+    assert_equal %w[db app], timing.map(&:name)
+    assert_equal 0, timing.invalid_count
+  end
+
+  def test_distinguishes_an_absent_header_from_an_empty_header
+    refute Crawlscope::ServerTiming.new(nil).present?
+
+    timing = Crawlscope::ServerTiming.new("")
+
+    assert timing.present?
+    assert_empty timing
+    assert_equal 0, timing.invalid_count
+  end
+end

data/test/crawlscope/sitemap_test.rb

@@ -34,6 +34,27 @@ class CrawlscopeSitemapTest < Minitest::Test
     assert_equal ["https://www.example.com/", "https://www.example.com/pricing"], parser.urls(base_url: "https://www.example.com")
   end
 
+  def test_remote_sitemap_sends_profile_token
+    request = stub_request(:get, "https://www.example.com/sitemap.xml")
+      .with(headers: {"X-Profile-Token" => "profile-token"})
+      .to_return(
+        status: 200,
+        body: <<~XML
+          <?xml version="1.0" encoding="UTF-8"?>
+          <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+            <url><loc>https://www.example.com/</loc></url>
+          </urlset>
+        XML
+      )
+
+    Crawlscope::Sitemap.new(
+      path: "https://www.example.com/sitemap.xml",
+      profile_token: "profile-token"
+    ).urls(base_url: "https://www.example.com")
+
+    assert_requested request
+  end
+
   def test_parses_remote_sitemap_index_with_child_sitemap
     stub_request(:get, "https://www.example.com/sitemap.xml")
       .to_return(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -238,6 +238,7 @@ files:
 - lib/crawlscope/railtie.rb
 - lib/crawlscope/rake_tasks.rb
 - lib/crawlscope/reporter.rb
+- lib/crawlscope/request_headers.rb
 - lib/crawlscope/result.rb
 - lib/crawlscope/rule_registry.rb
 - lib/crawlscope/rules/content_quality.rb
@@ -249,6 +250,9 @@ files:
 - lib/crawlscope/run.rb
 - lib/crawlscope/schema_registry.rb
 - lib/crawlscope/schemas.rb
+- lib/crawlscope/server_timing.rb
+- lib/crawlscope/server_timing/reporter.rb
+- lib/crawlscope/server_timing/summary.rb
 - lib/crawlscope/sitemap.rb
 - lib/crawlscope/structured_data/audit.rb
 - lib/crawlscope/structured_data/check.rb
@@ -281,6 +285,8 @@ files:
 - test/crawlscope/rule_registry_test.rb
 - test/crawlscope/run_test.rb
 - test/crawlscope/schema_registry_test.rb
+- test/crawlscope/server_timing_summary_test.rb
+- test/crawlscope/server_timing_test.rb
 - test/crawlscope/sitemap_test.rb
 - test/crawlscope/structured_data_audit_test.rb
 - test/crawlscope/structured_data_document_test.rb