crawlscope 0.7.2 → 0.8.1

data/test/crawlscope/structured_data_error_formatter_test.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class CrawlscopeStructuredDataErrorFormatterTest < Minitest::Test
+  def test_formats_missing_root_and_nested_fields_from_real_schema_errors
+    registry = Crawlscope::SchemaRegistry.default
+    errors = registry.validate({"@type" => "WebApplication", "name" => "App", "review" => {"@type" => "Review"}})
+
+    assert_equal "WebApplication.review/reviewRating: is required", format_error(errors.first)
+    assert_equal "Article.headline: is required", format_error(registry.validate({"@type" => "Article"}).first)
+  end
+
+  def test_preserves_unknown_constraints_without_schema_noise
+    error = {"field" => "price", "type" => "Product", "issue" => "The property '#/price' must satisfy a custom constraint in schema 3bdea3a8-461f-5eae-9d73-3243b48b4bc8"}
+
+    assert_equal "Product.price: must satisfy a custom constraint", format_error(error)
+    assert_equal "$: Unknown error", format_error({})
+  end
+
+  def test_formats_custom_minimum_length_constraint
+    registry = Crawlscope::SchemaRegistry.new(schemas: {"Thing" => {type: "object", properties: {name: {type: "string", minLength: 3}}}})
+    error = registry.validate({"@type" => "Thing", "name" => "x"}).first
+
+    assert_equal "Thing.name: 1 character; minimum 3", format_error(error)
+  end
+
+  def test_formats_measured_headline_length_with_symbol_or_json_keys
+    error = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "é" * 111}).first
+
+    assert_equal "Article.headline: 111 characters; maximum 110", format_error(error)
+    assert_equal "Article.headline: 111 characters; maximum 110", format_error(JSON.parse(JSON.generate(error)))
+  end
+
+  def test_formats_legacy_length_errors_without_a_measurement
+    error = {type: "Article", field: "headline", issue: "The property '#/headline' was not of a maximum string length of 110"}
+
+    assert_equal "Article.headline: exceeds 110 characters", format_error(error)
+  end
+
+  private
+
+  def format_error(error)
+    Crawlscope::StructuredData::ErrorFormatter.format(error)
+  end
+end

data/test/crawlscope/structured_data_reporter_test.rb

@@ -29,4 +29,30 @@ class CrawlscopeStructuredDataReporterTest < Minitest::Test
     assert_includes output, "headline: is required"
     assert_includes output, "/tmp/structured_data_report.json"
   end
+
+  def test_formats_real_schema_errors_in_summary_and_detailed_reports
+    errors = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "x" * 111})
+    result = Crawlscope::StructuredData::Audit::Outcome.new(entries: [
+      Crawlscope::StructuredData::Audit::Page.new(
+        url: "https://example.com/article", status: 200,
+        structured_items: [{source: "json-ld", data: {"@type" => "Article"}}],
+        errors: [{type: "Article", source: "json-ld", errors: errors}],
+        fetch_error: nil, content_type: "text/html", skipped_reason: nil
+      )
+    ])
+    io = StringIO.new
+    reporter = Crawlscope::StructuredData::Reporter.new(io: io)
+
+    reporter.report(result)
+    assert_includes io.string, "JSON-LD:\n      - Article.headline: 111 characters; maximum 110"
+    refute_includes io.string, "in schema"
+
+    io.truncate(0)
+    io.rewind
+    reporter.details(result, debug: false, renderer: :http)
+    assert_includes io.string, "Validation: FAILED\n  JSON-LD:\n    - Article.headline: 111 characters; maximum 110"
+    refute_includes io.string, "=" * 80
+    refute_includes io.string, "in schema"
+    assert_includes errors.first[:issue], "in schema"
+  end
 end

data/test/crawlscope/structured_data_rule_test.rb

@@ -31,6 +31,8 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
 
     assert_equal [:structured_data_schema_error], issues.to_a.map(&:code)
     assert_includes issues.to_a.first.message, "headline"
+    assert_equal "JSON-LD Article.headline: is required", issues.to_a.first.message
+    assert_includes issues.to_a.first.details[:errors].first[:issue], "in schema"
   end
 
   def test_reports_parse_errors_for_invalid_json_ld

data/test/crawlscope/uniqueness_rule_test.rb

@@ -44,6 +44,49 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     assert_operator issue.details[:similarity], :>=, issue.details[:threshold]
   end
 
+  def test_does_not_report_redirect_aliases_as_duplicate_pages
+    pages = [
+      page(url: "https://example.com/old", final_url: "https://example.com/current"),
+      page(url: "https://example.com/alias", final_url: "https://example.com/current"),
+      page(url: "https://example.com/current")
+    ]
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Uniqueness.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+
+    assert_empty issues.to_a
+  end
+
+  def test_reports_direct_duplicate_pages_without_including_redirect_aliases
+    pages = [
+      page(url: "https://example.com/old", final_url: "https://example.com/a"),
+      page(url: "https://example.com/a"),
+      page(url: "https://example.com/b")
+    ]
+    issues = Crawlscope::IssueCollection.new
+
+    Crawlscope::Rules::Uniqueness.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+
+    assert_equal 4, issues.size
+    issues.each do |issue|
+      assert_equal ["https://example.com/a", "https://example.com/b"], issue.details[:urls]
+    end
+  end
+
+  def test_excludes_redirects_from_near_duplicate_checks_and_scan_limit
+    pages = [
+      page(url: "https://example.com/old", final_url: "https://example.com/other", content: near_duplicate_content("reliable")),
+      page(url: "https://example.com/current", content: near_duplicate_content("dependable"))
+    ]
+
+    [Crawlscope::Rules::Uniqueness.new, Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)].each do |rule|
+      issues = Crawlscope::IssueCollection.new
+      rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
+
+      assert_empty issues.to_a
+    end
+  end
+
   def test_skips_near_duplicate_scan_when_page_count_exceeds_limit
     issues = Crawlscope::IssueCollection.new
     rule = Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)
@@ -72,7 +115,7 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     TEXT
   end
 
-  def page(url:, content: nil, canonical: nil)
+  def page(url:, content: nil, canonical: nil, final_url: url)
     repeated_text = content || ("Useful content " * 30).strip
     canonical_tag = canonical ? %(<link rel="canonical" href="#{canonical}">) : ""
     body = <<~HTML
@@ -91,8 +134,8 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
     Crawlscope::Page.new(
       url: url,
       normalized_url: url,
-      final_url: url,
-      normalized_final_url: url,
+      final_url: final_url,
+      normalized_final_url: final_url,
       status: 200,
       headers: {"content-type" => "text/html"},
       body: body,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlscope
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.1
 platform: ruby
 authors:
 - Paulo Fidalgo
@@ -238,6 +238,7 @@ files:
 - lib/crawlscope/railtie.rb
 - lib/crawlscope/rake_tasks.rb
 - lib/crawlscope/reporter.rb
+- lib/crawlscope/request_headers.rb
 - lib/crawlscope/result.rb
 - lib/crawlscope/rule_registry.rb
 - lib/crawlscope/rules/content_quality.rb
@@ -249,10 +250,14 @@ files:
 - lib/crawlscope/run.rb
 - lib/crawlscope/schema_registry.rb
 - lib/crawlscope/schemas.rb
+- lib/crawlscope/server_timing.rb
+- lib/crawlscope/server_timing/reporter.rb
+- lib/crawlscope/server_timing/summary.rb
 - lib/crawlscope/sitemap.rb
 - lib/crawlscope/structured_data/audit.rb
 - lib/crawlscope/structured_data/check.rb
 - lib/crawlscope/structured_data/document.rb
+- lib/crawlscope/structured_data/error_formatter.rb
 - lib/crawlscope/structured_data/report.rb
 - lib/crawlscope/structured_data/reporter.rb
 - lib/crawlscope/structured_data/writer.rb
@@ -281,9 +286,12 @@ files:
 - test/crawlscope/rule_registry_test.rb
 - test/crawlscope/run_test.rb
 - test/crawlscope/schema_registry_test.rb
+- test/crawlscope/server_timing_summary_test.rb
+- test/crawlscope/server_timing_test.rb
 - test/crawlscope/sitemap_test.rb
 - test/crawlscope/structured_data_audit_test.rb
 - test/crawlscope/structured_data_document_test.rb
+- test/crawlscope/structured_data_error_formatter_test.rb
 - test/crawlscope/structured_data_report_test.rb
 - test/crawlscope/structured_data_reporter_test.rb
 - test/crawlscope/structured_data_rule_test.rb
@@ -322,7 +330,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Audit sitemap URLs for metadata, structured data, uniqueness, and links
 test_files: []