crawlscope 0.8.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -58,11 +58,11 @@ class CrawlscopeReporterTest < Minitest::Test
58
58
  assert_includes output, "Status: WARNINGS"
59
59
  refute_includes output, "Status: FAILED"
60
60
  assert_includes output, "Issues: 5 total (5 warnings)"
61
- assert_includes output, "Summary:"
62
- assert_includes output, "links / low_dofollow_inlinks: 4"
63
- assert_includes output, " - /page-1 inbound 1/2 sources: /source-1"
64
- assert_includes output, " - /page-4 inbound 1/2 sources: /source-4"
65
- assert_includes output, "metadata / missing_title: 1"
61
+ refute_includes output, "Summary:"
62
+ assert_includes output, "Links: Too few incoming links without nofollow (4 warnings)"
63
+ assert_includes output, " - /page-1 incoming links without nofollow: 1; minimum 2\n linked from: /source-1"
64
+ assert_includes output, " - /page-4 incoming links without nofollow: 1; minimum 2\n linked from: /source-4"
65
+ assert_includes output, "Metadata: Missing title (1 warning)"
66
66
  refute_includes output, "Severity:"
67
67
  refute_includes output, "Category:"
68
68
  refute_includes output, "... 1 more"
@@ -90,7 +90,7 @@ class CrawlscopeReporterTest < Minitest::Test
90
90
  assert_includes output, "Issues: 2 total (1 error, 1 warning)"
91
91
  end
92
92
 
93
- def test_limits_large_issue_groups
93
+ def test_reports_every_issue_in_large_groups
94
94
  io = StringIO.new
95
95
  issues = Crawlscope::IssueCollection.new
96
96
  21.times do |index|
@@ -116,10 +116,10 @@ class CrawlscopeReporterTest < Minitest::Test
116
116
 
117
117
  output = io.string
118
118
 
119
- assert_includes output, "links / low_dofollow_inlinks: 21"
120
- assert_includes output, " - /page-20 inbound 1/2"
121
- refute_includes output, " - /page-21"
122
- assert_includes output, " ... 1 more"
119
+ assert_includes output, "Links: Too few incoming links without nofollow (21 warnings)"
120
+ assert_includes output, " - /page-20 incoming links without nofollow: 1; minimum 2"
121
+ assert_includes output, " - /page-21"
122
+ refute_includes output, " ... 1 more"
123
123
  end
124
124
 
125
125
  def test_reports_ratio_with_enough_precision_to_show_threshold_difference
@@ -144,7 +144,7 @@ class CrawlscopeReporterTest < Minitest::Test
144
144
 
145
145
  Crawlscope::Reporter.new(io: io).report(result)
146
146
 
147
- assert_includes io.string, "ratio 0.249/0.250"
147
+ assert_includes io.string, "unique words: 24.9%; minimum 25%"
148
148
  end
149
149
 
150
150
  def test_reports_source_details_on_one_line
@@ -173,9 +173,9 @@ class CrawlscopeReporterTest < Minitest::Test
173
173
 
174
174
  output = io.string
175
175
 
176
- assert_includes output, "sitemaps / indexable_page_missing_from_sitemap: 4"
177
- assert_includes output, " - /overview-1 indexable internal page is missing from sitemap source: /source-1"
178
- assert_includes output, " - /overview-4 indexable internal page is missing from sitemap source: /source-4"
176
+ assert_includes output, "Sitemaps: Linked pages missing from sitemap (4 warnings)"
177
+ assert_includes output, " - /overview-1 linked page is missing from sitemap\n linked from: /source-1"
178
+ assert_includes output, " - /overview-4 linked page is missing from sitemap\n linked from: /source-4"
179
179
  end
180
180
 
181
181
  def test_reports_server_timing_coverage_percentiles_signals_and_worst_pages
@@ -207,17 +207,188 @@ class CrawlscopeReporterTest < Minitest::Test
207
207
  output = io.string
208
208
  assert_includes output, "Server Timing:"
209
209
  assert_includes output, "Coverage: 3/3 pages (100.0%); durations on 2 pages"
210
- assert_includes output, "Duration metrics (dur; milliseconds assumed):"
210
+ assert_includes output, "Response times (ms):"
211
211
  assert_includes output, "total: 2 samples / 2 pages; avg 200ms; p50 200ms; p95 290ms; max 300ms"
212
212
  assert_includes output, 'db "Primary": 1 sample / 1 page'
213
213
  assert_includes output, 'cache "HIT": 1 sample / 1 page'
214
- assert_includes output, "Worst pages:"
214
+ assert_includes output, "Pages above 50 ms:"
215
215
  assert_includes output, "/slow: 300ms (total)"
216
216
  assert_includes output, "Ignored malformed entries: 1"
217
217
  end
218
218
 
219
+ def test_reports_redirect_destination_once_and_keeps_external_origins
220
+ [:sitemap_redirect_url, :redirected_page, :internal_link_redirects].each do |code|
221
+ output = report_issues({code: code, details: {final_url: "https://example.com/new?lang=en#top", status: 200}})
222
+
223
+ assert_includes output, "redirects to /new?lang=en#top"
224
+ assert_equal 1, output.scan("/new?lang=en#top").size
225
+ refute_includes output, "status 200"
226
+ refute_includes output, "final:"
227
+ end
228
+
229
+ ["https://other.example/new", "http://example.com/new", "https://example.com:8443/new"].each do |destination|
230
+ output = report_issues({code: :sitemap_redirect_url, details: {final_url: destination, status: 200}})
231
+ assert_includes output, "redirects to #{destination}"
232
+ end
233
+ end
234
+
235
+ def test_reports_canonical_target_and_explains_sitemap_mismatch
236
+ ["https://example.com/preferred?lang=en", "/preferred?lang=en"].each do |canonical|
237
+ output = report_issues({code: :non_canonical_page_in_sitemap, category: :sitemaps, details: {canonical: canonical}})
238
+ assert_includes output, "/page sitemap lists this URL, but its canonical points to /preferred?lang=en"
239
+ end
240
+
241
+ output = report_issues({code: :canonical_mismatch, details: {canonical: "https://other.example/page"}})
242
+ assert_includes output, "canonical points to a different URL: https://other.example/page"
243
+ end
244
+
245
+ def test_reports_real_schema_errors_on_separate_lines_without_raw_json
246
+ errors = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "x" * 111, "author" => "Name"})
247
+ output = report_issues({code: :structured_data_schema_error, category: :structured_data, details: {source: "json-ld", errors: errors}})
248
+
249
+ assert_includes output, "Structured data: Schema errors (1 warning)"
250
+ assert_includes output, " - /page JSON-LD\n Article.headline: 111 characters; maximum 110\n Article.author: must be object (got string)"
251
+ refute_includes output, "in schema"
252
+ refute_includes output, '"field":'
253
+ assert_includes errors.first[:issue], "in schema"
254
+ end
255
+
256
+ def test_reports_link_failures_without_repeating_status_or_sources
257
+ output = report_issues(
258
+ {code: :broken_internal_link, message: "broken internal link (HTTP 404, sources: https://example.com/source)", details: {status: 404, source_urls: ["https://example.com/source"]}},
259
+ {code: :unresolved_internal_link, details: {error: "timeout", source_urls: ["https://example.com/source"]}},
260
+ {code: :internal_link_redirects, details: {final_url: "https://example.com/new", source_urls: ["https://example.com/source"], status: 200}}
261
+ )
262
+
263
+ assert_includes output, "linked URL returns HTTP 404\n linked from: /source"
264
+ assert_equal 1, output.scan("404").size
265
+ assert_includes output, "cannot check linked URL: timeout\n linked from: /source"
266
+ assert_includes output, "redirects to /new (final HTTP 200)\n linked from: /source"
267
+ refute_includes output, "https://example.com/source"
268
+ end
269
+
270
+ def test_reports_canonical_status_with_both_relevant_urls
271
+ output = report_issues(
272
+ {code: :canonical_points_to_redirect, details: {canonical: "https://example.com/old", final_url: "https://example.com/new", status: 200}},
273
+ {code: :canonical_points_to_error, details: {canonical: "https://example.com/missing", status: 404}},
274
+ {code: :canonical_no_internal_inlinks, message: "canonical URL has no incoming internal links without nofollow", details: {source_url: "https://example.com/article"}}
275
+ )
276
+
277
+ assert_includes output, "canonical /old redirects to /new"
278
+ assert_includes output, "canonical /missing returns HTTP 404"
279
+ assert_includes output, " canonical on: /article"
280
+ refute_includes output, "linked from: /article"
281
+ end
282
+
283
+ def test_reports_errors_before_larger_warning_groups
284
+ warnings = 3.times.map { |index| {code: :missing_title, category: :metadata, url: "https://example.com/#{index}"} }
285
+ output = report_issues(*warnings, {code: :fetch_failed, category: :crawl, severity: :error, message: "timeout"})
286
+
287
+ assert_operator output.index("Crawl: Fetch failed"), :<, output.index("Metadata: Missing title")
288
+ refute_includes output, "metadata / missing_title"
289
+ end
290
+
291
+ def test_reports_full_duplicate_values_and_every_url
292
+ urls = 5.times.map { |index| "https://example.com/page-#{index}" }
293
+ value = "A long description " * 20
294
+ details = {value: value, urls: urls}
295
+ output = report_issues({code: :duplicate_meta_description, category: :uniqueness, details: details})
296
+
297
+ assert_includes output, "same meta description:"
298
+ assert_includes output, value
299
+ urls.each { |url| assert_includes output, " page: #{URI(url).path}" }
300
+ refute_includes output, "more)"
301
+ assert_equal value, details[:value]
302
+ assert_equal 5, details[:urls].size
303
+ end
304
+
305
+ def test_reports_measurements_with_units_and_limits
306
+ output = report_issues(
307
+ {code: :low_visible_text_ratio, details: {ratio: 0.079, threshold: 0.08}},
308
+ {code: :thin_visible_text, details: {word_count: 40, minimum: 250}},
309
+ {code: :near_duplicate_content, details: {similarity: 0.925, threshold: 0.9, urls: ["https://example.com/a", "https://example.com/b"]}},
310
+ {code: :near_duplicate_scan_skipped, details: {page_count: 300, max_pages: 250}}
311
+ )
312
+
313
+ assert_includes output, "visible text / HTML bytes: 7.9%; minimum 8%"
314
+ assert_includes output, "visible words: 40; minimum 250"
315
+ assert_includes output, "text similarity: 92.5%; threshold 90%\n page: /a\n page: /b"
316
+ assert_includes output, "similar-page check skipped: 300 pages; limit 250"
317
+ end
318
+
319
+ def test_reports_all_source_and_target_urls_on_labeled_lines
320
+ sources = 6.times.map { |index| "https://example.com/source-#{index}" }
321
+ targets = 6.times.map { |index| "https://example.com/target-#{index}" }
322
+ output = report_issues({code: :nofollow_internal_outlinks, details: {source_url: sources.first, source_urls: sources, target_urls: targets}})
323
+
324
+ sources.each { |url| assert_includes output, " linked from: #{URI(url).path}\n" }
325
+ targets.each { |url| assert_includes output, " links to: #{URI(url).path}\n" }
326
+ assert_equal 1, output.scan("linked from: /source-0").size
327
+ refute_includes output, "more"
328
+ end
329
+
330
+ def test_reports_all_timed_pages_and_full_descriptions
331
+ description = "A detailed metric description " * 5
332
+ pages = 12.times.map { |index| page("/timed-#{index}", server_timing: %(total;dur=#{index + 60};desc="#{description}")) }
333
+ result = Crawlscope::Result.new(base_url: "https://example.com", sitemap_path: "/sitemap.xml", urls: pages.map(&:url), pages: pages, issues: Crawlscope::IssueCollection.new)
334
+ io = StringIO.new
335
+ Crawlscope::Reporter.new(io: io).report(result)
336
+
337
+ pages.each { |item| assert_includes io.string, "#{URI(item.url).path}:" }
338
+ assert_includes io.string, description
339
+ end
340
+
341
+ def test_length_rows_do_not_repeat_the_field_or_limit_from_the_heading
342
+ output = report_issues(
343
+ {code: :meta_description_too_long, category: :metadata, message: "meta description: 184 characters; maximum 160", details: {length: 184}},
344
+ {code: :meta_description_too_short, category: :metadata, details: {length: 40, minimum: 110}},
345
+ {code: :title_too_long, category: :metadata, details: {length: 80}}
346
+ )
347
+
348
+ assert_includes output, "Metadata: Meta description exceeds 160 characters (1 warning)\n - /page 184 characters"
349
+ assert_includes output, "Metadata: Meta description below 110 characters (1 warning)\n - /page 40 characters"
350
+ assert_includes output, "Metadata: Title exceeds 72 characters (1 warning)\n - /page 80 characters"
351
+ refute_includes output, "meta description:"
352
+ refute_includes output, "maximum 160"
353
+ end
354
+
355
+ def test_lists_only_pages_strictly_above_50_milliseconds
356
+ pages = [page("/fast", server_timing: "total;dur=49"), page("/boundary", server_timing: "total;dur=50"), page("/slow", server_timing: "total;dur=50.001")]
357
+ result = Crawlscope::Result.new(base_url: "https://example.com", sitemap_path: "/sitemap.xml", urls: pages.map(&:url), pages: pages, issues: Crawlscope::IssueCollection.new)
358
+ io = StringIO.new
359
+ Crawlscope::Reporter.new(io: io).report(result)
360
+
361
+ assert_includes io.string, "Pages above 50 ms:\n /slow: 50.001ms (total)"
362
+ refute_includes io.string, "/fast:"
363
+ refute_includes io.string, "/boundary:"
364
+ assert_includes io.string, "Coverage: 3/3 pages"
365
+ assert_includes io.string, "total: 3 samples / 3 pages"
366
+ end
367
+
368
+ def test_omits_slow_page_section_when_all_pages_are_at_or_below_50_milliseconds
369
+ pages = [page("/boundary", server_timing: "total;dur=50")]
370
+ result = Crawlscope::Result.new(base_url: "https://example.com", sitemap_path: "/sitemap.xml", urls: pages.map(&:url), pages: pages, issues: Crawlscope::IssueCollection.new)
371
+ io = StringIO.new
372
+ Crawlscope::Reporter.new(io: io).report(result)
373
+
374
+ refute_includes io.string, "Pages above"
375
+ assert_includes io.string, "Server Timing:"
376
+ end
377
+
219
378
  private
220
379
 
380
+ def report_issues(*attributes, base_url: "https://example.com")
381
+ issues = Crawlscope::IssueCollection.new
382
+ attributes.each do |values|
383
+ defaults = {severity: :warning, category: :links, url: "#{base_url}/page", message: "original message", details: {}}
384
+ issues.add(**defaults.merge(values))
385
+ end
386
+ result = Crawlscope::Result.new(base_url: base_url, sitemap_path: "/sitemap.xml", urls: [], pages: [], issues: issues)
387
+ io = StringIO.new
388
+ Crawlscope::Reporter.new(io: io).report(result)
389
+ io.string
390
+ end
391
+
221
392
  def page(path = "/", server_timing: nil)
222
393
  url = "https://example.com#{path}"
223
394
  headers = {}
@@ -185,7 +185,7 @@ class CrawlscopeRunTest < Minitest::Test
185
185
  assert_equal ["https://example.com"], JSON.parse(File.read(report_path)).fetch("results").keys
186
186
  assert_includes output.string, "JavaScript mode enabled (Ferrum)"
187
187
  assert_includes output.string, "Validating JSON-LD on 1 URL(s)"
188
- assert_includes output.string, "All valid!"
188
+ assert_includes output.string, "Validation: OK"
189
189
  assert_includes output.string, "All 1 URLs passed validation."
190
190
  ensure
191
191
  FileUtils.rm_rf(report_dir) if report_dir
@@ -46,6 +46,39 @@ class CrawlscopeSchemaRegistryTest < Minitest::Test
46
46
  assert registry.registered?("VideoObject")
47
47
  end
48
48
 
49
+ def test_length_errors_include_character_count_without_copying_the_value
50
+ error = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "é" * 111}).first
51
+
52
+ assert_equal 111, error[:length]
53
+ assert_equal "headline", error[:field]
54
+ assert_includes error[:issue], "maximum string length of 110"
55
+ assert_equal [:field, :issue, :type, :length], error.keys
56
+ end
57
+
58
+ def test_measures_string_fields_inside_nested_arrays
59
+ schema = {
60
+ type: "object",
61
+ properties: {
62
+ entries: {type: "array", items: {type: "object", properties: {name: {type: "string", maxLength: 3}}}}
63
+ }
64
+ }
65
+ registry = Crawlscope::SchemaRegistry.new(schemas: {"Thing" => schema})
66
+ errors = registry.validate({"@type" => "Thing", "entries" => [{"name" => "ok"}, {"name" => "éééé"}]})
67
+
68
+ assert_equal 1, errors.size
69
+ assert_equal "entries/1/name", errors.first[:field]
70
+ assert_equal 4, errors.first[:length]
71
+ end
72
+
73
+ def test_measures_each_graph_items_headline_independently
74
+ errors = Crawlscope::SchemaRegistry.default.validate({"@graph" => [
75
+ {"@type" => "Article", "headline" => "a" * 111},
76
+ {"@type" => "Article", "headline" => "b" * 125}
77
+ ]})
78
+
79
+ assert_equal [111, 125], errors.map { |error| error[:length] }
80
+ end
81
+
49
82
  def test_web_application_review_requires_review_rating
50
83
  errors = Crawlscope::SchemaRegistry.default.validate(
51
84
  {
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "test_helper"
4
+
5
+ class CrawlscopeStructuredDataErrorFormatterTest < Minitest::Test
6
+ def test_formats_missing_root_and_nested_fields_from_real_schema_errors
7
+ registry = Crawlscope::SchemaRegistry.default
8
+ errors = registry.validate({"@type" => "WebApplication", "name" => "App", "review" => {"@type" => "Review"}})
9
+
10
+ assert_equal "WebApplication.review/reviewRating: is required", format_error(errors.first)
11
+ assert_equal "Article.headline: is required", format_error(registry.validate({"@type" => "Article"}).first)
12
+ end
13
+
14
+ def test_preserves_unknown_constraints_without_schema_noise
15
+ error = {"field" => "price", "type" => "Product", "issue" => "The property '#/price' must satisfy a custom constraint in schema 3bdea3a8-461f-5eae-9d73-3243b48b4bc8"}
16
+
17
+ assert_equal "Product.price: must satisfy a custom constraint", format_error(error)
18
+ assert_equal "$: Unknown error", format_error({})
19
+ end
20
+
21
+ def test_formats_custom_minimum_length_constraint
22
+ registry = Crawlscope::SchemaRegistry.new(schemas: {"Thing" => {type: "object", properties: {name: {type: "string", minLength: 3}}}})
23
+ error = registry.validate({"@type" => "Thing", "name" => "x"}).first
24
+
25
+ assert_equal "Thing.name: 1 character; minimum 3", format_error(error)
26
+ end
27
+
28
+ def test_formats_measured_headline_length_with_symbol_or_json_keys
29
+ error = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "é" * 111}).first
30
+
31
+ assert_equal "Article.headline: 111 characters; maximum 110", format_error(error)
32
+ assert_equal "Article.headline: 111 characters; maximum 110", format_error(JSON.parse(JSON.generate(error)))
33
+ end
34
+
35
+ def test_formats_legacy_length_errors_without_a_measurement
36
+ error = {type: "Article", field: "headline", issue: "The property '#/headline' was not of a maximum string length of 110"}
37
+
38
+ assert_equal "Article.headline: exceeds 110 characters", format_error(error)
39
+ end
40
+
41
+ private
42
+
43
+ def format_error(error)
44
+ Crawlscope::StructuredData::ErrorFormatter.format(error)
45
+ end
46
+ end
@@ -29,4 +29,30 @@ class CrawlscopeStructuredDataReporterTest < Minitest::Test
29
29
  assert_includes output, "headline: is required"
30
30
  assert_includes output, "/tmp/structured_data_report.json"
31
31
  end
32
+
33
+ def test_formats_real_schema_errors_in_summary_and_detailed_reports
34
+ errors = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "x" * 111})
35
+ result = Crawlscope::StructuredData::Audit::Outcome.new(entries: [
36
+ Crawlscope::StructuredData::Audit::Page.new(
37
+ url: "https://example.com/article", status: 200,
38
+ structured_items: [{source: "json-ld", data: {"@type" => "Article"}}],
39
+ errors: [{type: "Article", source: "json-ld", errors: errors}],
40
+ fetch_error: nil, content_type: "text/html", skipped_reason: nil
41
+ )
42
+ ])
43
+ io = StringIO.new
44
+ reporter = Crawlscope::StructuredData::Reporter.new(io: io)
45
+
46
+ reporter.report(result)
47
+ assert_includes io.string, "JSON-LD:\n - Article.headline: 111 characters; maximum 110"
48
+ refute_includes io.string, "in schema"
49
+
50
+ io.truncate(0)
51
+ io.rewind
52
+ reporter.details(result, debug: false, renderer: :http)
53
+ assert_includes io.string, "Validation: FAILED\n JSON-LD:\n - Article.headline: 111 characters; maximum 110"
54
+ refute_includes io.string, "=" * 80
55
+ refute_includes io.string, "in schema"
56
+ assert_includes errors.first[:issue], "in schema"
57
+ end
32
58
  end
@@ -31,6 +31,8 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
31
31
 
32
32
  assert_equal [:structured_data_schema_error], issues.to_a.map(&:code)
33
33
  assert_includes issues.to_a.first.message, "headline"
34
+ assert_equal "JSON-LD Article.headline: is required", issues.to_a.first.message
35
+ assert_includes issues.to_a.first.details[:errors].first[:issue], "in schema"
34
36
  end
35
37
 
36
38
  def test_reports_parse_errors_for_invalid_json_ld
@@ -44,6 +44,49 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
44
44
  assert_operator issue.details[:similarity], :>=, issue.details[:threshold]
45
45
  end
46
46
 
47
+ def test_does_not_report_redirect_aliases_as_duplicate_pages
48
+ pages = [
49
+ page(url: "https://example.com/old", final_url: "https://example.com/current"),
50
+ page(url: "https://example.com/alias", final_url: "https://example.com/current"),
51
+ page(url: "https://example.com/current")
52
+ ]
53
+ issues = Crawlscope::IssueCollection.new
54
+
55
+ Crawlscope::Rules::Uniqueness.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
56
+
57
+ assert_empty issues.to_a
58
+ end
59
+
60
+ def test_reports_direct_duplicate_pages_without_including_redirect_aliases
61
+ pages = [
62
+ page(url: "https://example.com/old", final_url: "https://example.com/a"),
63
+ page(url: "https://example.com/a"),
64
+ page(url: "https://example.com/b")
65
+ ]
66
+ issues = Crawlscope::IssueCollection.new
67
+
68
+ Crawlscope::Rules::Uniqueness.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
69
+
70
+ assert_equal 4, issues.size
71
+ issues.each do |issue|
72
+ assert_equal ["https://example.com/a", "https://example.com/b"], issue.details[:urls]
73
+ end
74
+ end
75
+
76
+ def test_excludes_redirects_from_near_duplicate_checks_and_scan_limit
77
+ pages = [
78
+ page(url: "https://example.com/old", final_url: "https://example.com/other", content: near_duplicate_content("reliable")),
79
+ page(url: "https://example.com/current", content: near_duplicate_content("dependable"))
80
+ ]
81
+
82
+ [Crawlscope::Rules::Uniqueness.new, Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)].each do |rule|
83
+ issues = Crawlscope::IssueCollection.new
84
+ rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
85
+
86
+ assert_empty issues.to_a
87
+ end
88
+ end
89
+
47
90
  def test_skips_near_duplicate_scan_when_page_count_exceeds_limit
48
91
  issues = Crawlscope::IssueCollection.new
49
92
  rule = Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)
@@ -72,7 +115,7 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
72
115
  TEXT
73
116
  end
74
117
 
75
- def page(url:, content: nil, canonical: nil)
118
+ def page(url:, content: nil, canonical: nil, final_url: url)
76
119
  repeated_text = content || ("Useful content " * 30).strip
77
120
  canonical_tag = canonical ? %(<link rel="canonical" href="#{canonical}">) : ""
78
121
  body = <<~HTML
@@ -91,8 +134,8 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
91
134
  Crawlscope::Page.new(
92
135
  url: url,
93
136
  normalized_url: url,
94
- final_url: url,
95
- normalized_final_url: url,
137
+ final_url: final_url,
138
+ normalized_final_url: final_url,
96
139
  status: 200,
97
140
  headers: {"content-type" => "text/html"},
98
141
  body: body,
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawlscope
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.8.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Paulo Fidalgo
@@ -257,6 +257,7 @@ files:
257
257
  - lib/crawlscope/structured_data/audit.rb
258
258
  - lib/crawlscope/structured_data/check.rb
259
259
  - lib/crawlscope/structured_data/document.rb
260
+ - lib/crawlscope/structured_data/error_formatter.rb
260
261
  - lib/crawlscope/structured_data/report.rb
261
262
  - lib/crawlscope/structured_data/reporter.rb
262
263
  - lib/crawlscope/structured_data/writer.rb
@@ -290,6 +291,7 @@ files:
290
291
  - test/crawlscope/sitemap_test.rb
291
292
  - test/crawlscope/structured_data_audit_test.rb
292
293
  - test/crawlscope/structured_data_document_test.rb
294
+ - test/crawlscope/structured_data_error_formatter_test.rb
293
295
  - test/crawlscope/structured_data_report_test.rb
294
296
  - test/crawlscope/structured_data_reporter_test.rb
295
297
  - test/crawlscope/structured_data_rule_test.rb
@@ -328,7 +330,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
328
330
  - !ruby/object:Gem::Version
329
331
  version: '0'
330
332
  requirements: []
331
- rubygems_version: 4.0.16
333
+ rubygems_version: 4.0.20
332
334
  specification_version: 4
333
335
  summary: Audit sitemap URLs for metadata, structured data, uniqueness, and links
334
336
  test_files: []