crawlscope 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +21 -10
- data/lib/crawlscope/crawl.rb +0 -1
- data/lib/crawlscope/reporter.rb +141 -63
- data/lib/crawlscope/rules/content_quality.rb +5 -5
- data/lib/crawlscope/rules/indexability.rb +1 -1
- data/lib/crawlscope/rules/links.rb +28 -34
- data/lib/crawlscope/rules/metadata.rb +25 -25
- data/lib/crawlscope/rules/structured_data.rb +13 -4
- data/lib/crawlscope/rules/uniqueness.rb +9 -7
- data/lib/crawlscope/schema_registry.rb +17 -1
- data/lib/crawlscope/server_timing/reporter.rb +6 -4
- data/lib/crawlscope/structured_data/error_formatter.rb +42 -0
- data/lib/crawlscope/structured_data/reporter.rb +10 -10
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/crawl_test.rb +36 -0
- data/test/crawlscope/links_rule_test.rb +40 -4
- data/test/crawlscope/metadata_rule_test.rb +37 -4
- data/test/crawlscope/reporter_test.rb +187 -16
- data/test/crawlscope/run_test.rb +1 -1
- data/test/crawlscope/schema_registry_test.rb +33 -0
- data/test/crawlscope/structured_data_error_formatter_test.rb +46 -0
- data/test/crawlscope/structured_data_reporter_test.rb +26 -0
- data/test/crawlscope/structured_data_rule_test.rb +2 -0
- data/test/crawlscope/uniqueness_rule_test.rb +46 -3
- metadata +4 -2
|
@@ -58,11 +58,11 @@ class CrawlscopeReporterTest < Minitest::Test
|
|
|
58
58
|
assert_includes output, "Status: WARNINGS"
|
|
59
59
|
refute_includes output, "Status: FAILED"
|
|
60
60
|
assert_includes output, "Issues: 5 total (5 warnings)"
|
|
61
|
-
|
|
62
|
-
assert_includes output, "links
|
|
63
|
-
assert_includes output, " - /page-1
|
|
64
|
-
assert_includes output, " - /page-4
|
|
65
|
-
assert_includes output, "
|
|
61
|
+
refute_includes output, "Summary:"
|
|
62
|
+
assert_includes output, "Links: Too few incoming links without nofollow (4 warnings)"
|
|
63
|
+
assert_includes output, " - /page-1 incoming links without nofollow: 1; minimum 2\n linked from: /source-1"
|
|
64
|
+
assert_includes output, " - /page-4 incoming links without nofollow: 1; minimum 2\n linked from: /source-4"
|
|
65
|
+
assert_includes output, "Metadata: Missing title (1 warning)"
|
|
66
66
|
refute_includes output, "Severity:"
|
|
67
67
|
refute_includes output, "Category:"
|
|
68
68
|
refute_includes output, "... 1 more"
|
|
@@ -90,7 +90,7 @@ class CrawlscopeReporterTest < Minitest::Test
|
|
|
90
90
|
assert_includes output, "Issues: 2 total (1 error, 1 warning)"
|
|
91
91
|
end
|
|
92
92
|
|
|
93
|
-
def
|
|
93
|
+
def test_reports_every_issue_in_large_groups
|
|
94
94
|
io = StringIO.new
|
|
95
95
|
issues = Crawlscope::IssueCollection.new
|
|
96
96
|
21.times do |index|
|
|
@@ -116,10 +116,10 @@ class CrawlscopeReporterTest < Minitest::Test
|
|
|
116
116
|
|
|
117
117
|
output = io.string
|
|
118
118
|
|
|
119
|
-
assert_includes output, "links
|
|
120
|
-
assert_includes output, " - /page-20
|
|
121
|
-
|
|
122
|
-
|
|
119
|
+
assert_includes output, "Links: Too few incoming links without nofollow (21 warnings)"
|
|
120
|
+
assert_includes output, " - /page-20 incoming links without nofollow: 1; minimum 2"
|
|
121
|
+
assert_includes output, " - /page-21"
|
|
122
|
+
refute_includes output, " ... 1 more"
|
|
123
123
|
end
|
|
124
124
|
|
|
125
125
|
def test_reports_ratio_with_enough_precision_to_show_threshold_difference
|
|
@@ -144,7 +144,7 @@ class CrawlscopeReporterTest < Minitest::Test
|
|
|
144
144
|
|
|
145
145
|
Crawlscope::Reporter.new(io: io).report(result)
|
|
146
146
|
|
|
147
|
-
assert_includes io.string, "
|
|
147
|
+
assert_includes io.string, "unique words: 24.9%; minimum 25%"
|
|
148
148
|
end
|
|
149
149
|
|
|
150
150
|
def test_reports_source_details_on_one_line
|
|
@@ -173,9 +173,9 @@ class CrawlscopeReporterTest < Minitest::Test
|
|
|
173
173
|
|
|
174
174
|
output = io.string
|
|
175
175
|
|
|
176
|
-
assert_includes output, "
|
|
177
|
-
assert_includes output, " - /overview-1
|
|
178
|
-
assert_includes output, " - /overview-4
|
|
176
|
+
assert_includes output, "Sitemaps: Linked pages missing from sitemap (4 warnings)"
|
|
177
|
+
assert_includes output, " - /overview-1 linked page is missing from sitemap\n linked from: /source-1"
|
|
178
|
+
assert_includes output, " - /overview-4 linked page is missing from sitemap\n linked from: /source-4"
|
|
179
179
|
end
|
|
180
180
|
|
|
181
181
|
def test_reports_server_timing_coverage_percentiles_signals_and_worst_pages
|
|
@@ -207,17 +207,188 @@ class CrawlscopeReporterTest < Minitest::Test
|
|
|
207
207
|
output = io.string
|
|
208
208
|
assert_includes output, "Server Timing:"
|
|
209
209
|
assert_includes output, "Coverage: 3/3 pages (100.0%); durations on 2 pages"
|
|
210
|
-
assert_includes output, "
|
|
210
|
+
assert_includes output, "Response times (ms):"
|
|
211
211
|
assert_includes output, "total: 2 samples / 2 pages; avg 200ms; p50 200ms; p95 290ms; max 300ms"
|
|
212
212
|
assert_includes output, 'db "Primary": 1 sample / 1 page'
|
|
213
213
|
assert_includes output, 'cache "HIT": 1 sample / 1 page'
|
|
214
|
-
assert_includes output, "
|
|
214
|
+
assert_includes output, "Pages above 50 ms:"
|
|
215
215
|
assert_includes output, "/slow: 300ms (total)"
|
|
216
216
|
assert_includes output, "Ignored malformed entries: 1"
|
|
217
217
|
end
|
|
218
218
|
|
|
219
|
+
def test_reports_redirect_destination_once_and_keeps_external_origins
|
|
220
|
+
[:sitemap_redirect_url, :redirected_page, :internal_link_redirects].each do |code|
|
|
221
|
+
output = report_issues({code: code, details: {final_url: "https://example.com/new?lang=en#top", status: 200}})
|
|
222
|
+
|
|
223
|
+
assert_includes output, "redirects to /new?lang=en#top"
|
|
224
|
+
assert_equal 1, output.scan("/new?lang=en#top").size
|
|
225
|
+
refute_includes output, "status 200"
|
|
226
|
+
refute_includes output, "final:"
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
["https://other.example/new", "http://example.com/new", "https://example.com:8443/new"].each do |destination|
|
|
230
|
+
output = report_issues({code: :sitemap_redirect_url, details: {final_url: destination, status: 200}})
|
|
231
|
+
assert_includes output, "redirects to #{destination}"
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def test_reports_canonical_target_and_explains_sitemap_mismatch
|
|
236
|
+
["https://example.com/preferred?lang=en", "/preferred?lang=en"].each do |canonical|
|
|
237
|
+
output = report_issues({code: :non_canonical_page_in_sitemap, category: :sitemaps, details: {canonical: canonical}})
|
|
238
|
+
assert_includes output, "/page sitemap lists this URL, but its canonical points to /preferred?lang=en"
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
output = report_issues({code: :canonical_mismatch, details: {canonical: "https://other.example/page"}})
|
|
242
|
+
assert_includes output, "canonical points to a different URL: https://other.example/page"
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def test_reports_real_schema_errors_on_separate_lines_without_raw_json
|
|
246
|
+
errors = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "x" * 111, "author" => "Name"})
|
|
247
|
+
output = report_issues({code: :structured_data_schema_error, category: :structured_data, details: {source: "json-ld", errors: errors}})
|
|
248
|
+
|
|
249
|
+
assert_includes output, "Structured data: Schema errors (1 warning)"
|
|
250
|
+
assert_includes output, " - /page JSON-LD\n Article.headline: 111 characters; maximum 110\n Article.author: must be object (got string)"
|
|
251
|
+
refute_includes output, "in schema"
|
|
252
|
+
refute_includes output, '"field":'
|
|
253
|
+
assert_includes errors.first[:issue], "in schema"
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def test_reports_link_failures_without_repeating_status_or_sources
|
|
257
|
+
output = report_issues(
|
|
258
|
+
{code: :broken_internal_link, message: "broken internal link (HTTP 404, sources: https://example.com/source)", details: {status: 404, source_urls: ["https://example.com/source"]}},
|
|
259
|
+
{code: :unresolved_internal_link, details: {error: "timeout", source_urls: ["https://example.com/source"]}},
|
|
260
|
+
{code: :internal_link_redirects, details: {final_url: "https://example.com/new", source_urls: ["https://example.com/source"], status: 200}}
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
assert_includes output, "linked URL returns HTTP 404\n linked from: /source"
|
|
264
|
+
assert_equal 1, output.scan("404").size
|
|
265
|
+
assert_includes output, "cannot check linked URL: timeout\n linked from: /source"
|
|
266
|
+
assert_includes output, "redirects to /new (final HTTP 200)\n linked from: /source"
|
|
267
|
+
refute_includes output, "https://example.com/source"
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def test_reports_canonical_status_with_both_relevant_urls
|
|
271
|
+
output = report_issues(
|
|
272
|
+
{code: :canonical_points_to_redirect, details: {canonical: "https://example.com/old", final_url: "https://example.com/new", status: 200}},
|
|
273
|
+
{code: :canonical_points_to_error, details: {canonical: "https://example.com/missing", status: 404}},
|
|
274
|
+
{code: :canonical_no_internal_inlinks, message: "canonical URL has no incoming internal links without nofollow", details: {source_url: "https://example.com/article"}}
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
assert_includes output, "canonical /old redirects to /new"
|
|
278
|
+
assert_includes output, "canonical /missing returns HTTP 404"
|
|
279
|
+
assert_includes output, " canonical on: /article"
|
|
280
|
+
refute_includes output, "linked from: /article"
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def test_reports_errors_before_larger_warning_groups
|
|
284
|
+
warnings = 3.times.map { |index| {code: :missing_title, category: :metadata, url: "https://example.com/#{index}"} }
|
|
285
|
+
output = report_issues(*warnings, {code: :fetch_failed, category: :crawl, severity: :error, message: "timeout"})
|
|
286
|
+
|
|
287
|
+
assert_operator output.index("Crawl: Fetch failed"), :<, output.index("Metadata: Missing title")
|
|
288
|
+
refute_includes output, "metadata / missing_title"
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def test_reports_full_duplicate_values_and_every_url
|
|
292
|
+
urls = 5.times.map { |index| "https://example.com/page-#{index}" }
|
|
293
|
+
value = "A long description " * 20
|
|
294
|
+
details = {value: value, urls: urls}
|
|
295
|
+
output = report_issues({code: :duplicate_meta_description, category: :uniqueness, details: details})
|
|
296
|
+
|
|
297
|
+
assert_includes output, "same meta description:"
|
|
298
|
+
assert_includes output, value
|
|
299
|
+
urls.each { |url| assert_includes output, " page: #{URI(url).path}" }
|
|
300
|
+
refute_includes output, "more)"
|
|
301
|
+
assert_equal value, details[:value]
|
|
302
|
+
assert_equal 5, details[:urls].size
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def test_reports_measurements_with_units_and_limits
|
|
306
|
+
output = report_issues(
|
|
307
|
+
{code: :low_visible_text_ratio, details: {ratio: 0.079, threshold: 0.08}},
|
|
308
|
+
{code: :thin_visible_text, details: {word_count: 40, minimum: 250}},
|
|
309
|
+
{code: :near_duplicate_content, details: {similarity: 0.925, threshold: 0.9, urls: ["https://example.com/a", "https://example.com/b"]}},
|
|
310
|
+
{code: :near_duplicate_scan_skipped, details: {page_count: 300, max_pages: 250}}
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
assert_includes output, "visible text / HTML bytes: 7.9%; minimum 8%"
|
|
314
|
+
assert_includes output, "visible words: 40; minimum 250"
|
|
315
|
+
assert_includes output, "text similarity: 92.5%; threshold 90%\n page: /a\n page: /b"
|
|
316
|
+
assert_includes output, "similar-page check skipped: 300 pages; limit 250"
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
def test_reports_all_source_and_target_urls_on_labeled_lines
|
|
320
|
+
sources = 6.times.map { |index| "https://example.com/source-#{index}" }
|
|
321
|
+
targets = 6.times.map { |index| "https://example.com/target-#{index}" }
|
|
322
|
+
output = report_issues({code: :nofollow_internal_outlinks, details: {source_url: sources.first, source_urls: sources, target_urls: targets}})
|
|
323
|
+
|
|
324
|
+
sources.each { |url| assert_includes output, " linked from: #{URI(url).path}\n" }
|
|
325
|
+
targets.each { |url| assert_includes output, " links to: #{URI(url).path}\n" }
|
|
326
|
+
assert_equal 1, output.scan("linked from: /source-0").size
|
|
327
|
+
refute_includes output, "more"
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
def test_reports_all_timed_pages_and_full_descriptions
|
|
331
|
+
description = "A detailed metric description " * 5
|
|
332
|
+
pages = 12.times.map { |index| page("/timed-#{index}", server_timing: %(total;dur=#{index + 60};desc="#{description}")) }
|
|
333
|
+
result = Crawlscope::Result.new(base_url: "https://example.com", sitemap_path: "/sitemap.xml", urls: pages.map(&:url), pages: pages, issues: Crawlscope::IssueCollection.new)
|
|
334
|
+
io = StringIO.new
|
|
335
|
+
Crawlscope::Reporter.new(io: io).report(result)
|
|
336
|
+
|
|
337
|
+
pages.each { |item| assert_includes io.string, "#{URI(item.url).path}:" }
|
|
338
|
+
assert_includes io.string, description
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def test_length_rows_do_not_repeat_the_field_or_limit_from_the_heading
|
|
342
|
+
output = report_issues(
|
|
343
|
+
{code: :meta_description_too_long, category: :metadata, message: "meta description: 184 characters; maximum 160", details: {length: 184}},
|
|
344
|
+
{code: :meta_description_too_short, category: :metadata, details: {length: 40, minimum: 110}},
|
|
345
|
+
{code: :title_too_long, category: :metadata, details: {length: 80}}
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
assert_includes output, "Metadata: Meta description exceeds 160 characters (1 warning)\n - /page 184 characters"
|
|
349
|
+
assert_includes output, "Metadata: Meta description below 110 characters (1 warning)\n - /page 40 characters"
|
|
350
|
+
assert_includes output, "Metadata: Title exceeds 72 characters (1 warning)\n - /page 80 characters"
|
|
351
|
+
refute_includes output, "meta description:"
|
|
352
|
+
refute_includes output, "maximum 160"
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
def test_lists_only_pages_strictly_above_50_milliseconds
|
|
356
|
+
pages = [page("/fast", server_timing: "total;dur=49"), page("/boundary", server_timing: "total;dur=50"), page("/slow", server_timing: "total;dur=50.001")]
|
|
357
|
+
result = Crawlscope::Result.new(base_url: "https://example.com", sitemap_path: "/sitemap.xml", urls: pages.map(&:url), pages: pages, issues: Crawlscope::IssueCollection.new)
|
|
358
|
+
io = StringIO.new
|
|
359
|
+
Crawlscope::Reporter.new(io: io).report(result)
|
|
360
|
+
|
|
361
|
+
assert_includes io.string, "Pages above 50 ms:\n /slow: 50.001ms (total)"
|
|
362
|
+
refute_includes io.string, "/fast:"
|
|
363
|
+
refute_includes io.string, "/boundary:"
|
|
364
|
+
assert_includes io.string, "Coverage: 3/3 pages"
|
|
365
|
+
assert_includes io.string, "total: 3 samples / 3 pages"
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def test_omits_slow_page_section_when_all_pages_are_at_or_below_50_milliseconds
|
|
369
|
+
pages = [page("/boundary", server_timing: "total;dur=50")]
|
|
370
|
+
result = Crawlscope::Result.new(base_url: "https://example.com", sitemap_path: "/sitemap.xml", urls: pages.map(&:url), pages: pages, issues: Crawlscope::IssueCollection.new)
|
|
371
|
+
io = StringIO.new
|
|
372
|
+
Crawlscope::Reporter.new(io: io).report(result)
|
|
373
|
+
|
|
374
|
+
refute_includes io.string, "Pages above"
|
|
375
|
+
assert_includes io.string, "Server Timing:"
|
|
376
|
+
end
|
|
377
|
+
|
|
219
378
|
private
|
|
220
379
|
|
|
380
|
+
def report_issues(*attributes, base_url: "https://example.com")
|
|
381
|
+
issues = Crawlscope::IssueCollection.new
|
|
382
|
+
attributes.each do |values|
|
|
383
|
+
defaults = {severity: :warning, category: :links, url: "#{base_url}/page", message: "original message", details: {}}
|
|
384
|
+
issues.add(**defaults.merge(values))
|
|
385
|
+
end
|
|
386
|
+
result = Crawlscope::Result.new(base_url: base_url, sitemap_path: "/sitemap.xml", urls: [], pages: [], issues: issues)
|
|
387
|
+
io = StringIO.new
|
|
388
|
+
Crawlscope::Reporter.new(io: io).report(result)
|
|
389
|
+
io.string
|
|
390
|
+
end
|
|
391
|
+
|
|
221
392
|
def page(path = "/", server_timing: nil)
|
|
222
393
|
url = "https://example.com#{path}"
|
|
223
394
|
headers = {}
|
data/test/crawlscope/run_test.rb
CHANGED
|
@@ -185,7 +185,7 @@ class CrawlscopeRunTest < Minitest::Test
|
|
|
185
185
|
assert_equal ["https://example.com"], JSON.parse(File.read(report_path)).fetch("results").keys
|
|
186
186
|
assert_includes output.string, "JavaScript mode enabled (Ferrum)"
|
|
187
187
|
assert_includes output.string, "Validating JSON-LD on 1 URL(s)"
|
|
188
|
-
assert_includes output.string, "
|
|
188
|
+
assert_includes output.string, "Validation: OK"
|
|
189
189
|
assert_includes output.string, "All 1 URLs passed validation."
|
|
190
190
|
ensure
|
|
191
191
|
FileUtils.rm_rf(report_dir) if report_dir
|
|
@@ -46,6 +46,39 @@ class CrawlscopeSchemaRegistryTest < Minitest::Test
|
|
|
46
46
|
assert registry.registered?("VideoObject")
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
+
def test_length_errors_include_character_count_without_copying_the_value
|
|
50
|
+
error = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "é" * 111}).first
|
|
51
|
+
|
|
52
|
+
assert_equal 111, error[:length]
|
|
53
|
+
assert_equal "headline", error[:field]
|
|
54
|
+
assert_includes error[:issue], "maximum string length of 110"
|
|
55
|
+
assert_equal [:field, :issue, :type, :length], error.keys
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def test_measures_string_fields_inside_nested_arrays
|
|
59
|
+
schema = {
|
|
60
|
+
type: "object",
|
|
61
|
+
properties: {
|
|
62
|
+
entries: {type: "array", items: {type: "object", properties: {name: {type: "string", maxLength: 3}}}}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
registry = Crawlscope::SchemaRegistry.new(schemas: {"Thing" => schema})
|
|
66
|
+
errors = registry.validate({"@type" => "Thing", "entries" => [{"name" => "ok"}, {"name" => "éééé"}]})
|
|
67
|
+
|
|
68
|
+
assert_equal 1, errors.size
|
|
69
|
+
assert_equal "entries/1/name", errors.first[:field]
|
|
70
|
+
assert_equal 4, errors.first[:length]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def test_measures_each_graph_items_headline_independently
|
|
74
|
+
errors = Crawlscope::SchemaRegistry.default.validate({"@graph" => [
|
|
75
|
+
{"@type" => "Article", "headline" => "a" * 111},
|
|
76
|
+
{"@type" => "Article", "headline" => "b" * 125}
|
|
77
|
+
]})
|
|
78
|
+
|
|
79
|
+
assert_equal [111, 125], errors.map { |error| error[:length] }
|
|
80
|
+
end
|
|
81
|
+
|
|
49
82
|
def test_web_application_review_requires_review_rating
|
|
50
83
|
errors = Crawlscope::SchemaRegistry.default.validate(
|
|
51
84
|
{
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class CrawlscopeStructuredDataErrorFormatterTest < Minitest::Test
|
|
6
|
+
def test_formats_missing_root_and_nested_fields_from_real_schema_errors
|
|
7
|
+
registry = Crawlscope::SchemaRegistry.default
|
|
8
|
+
errors = registry.validate({"@type" => "WebApplication", "name" => "App", "review" => {"@type" => "Review"}})
|
|
9
|
+
|
|
10
|
+
assert_equal "WebApplication.review/reviewRating: is required", format_error(errors.first)
|
|
11
|
+
assert_equal "Article.headline: is required", format_error(registry.validate({"@type" => "Article"}).first)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def test_preserves_unknown_constraints_without_schema_noise
|
|
15
|
+
error = {"field" => "price", "type" => "Product", "issue" => "The property '#/price' must satisfy a custom constraint in schema 3bdea3a8-461f-5eae-9d73-3243b48b4bc8"}
|
|
16
|
+
|
|
17
|
+
assert_equal "Product.price: must satisfy a custom constraint", format_error(error)
|
|
18
|
+
assert_equal "$: Unknown error", format_error({})
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def test_formats_custom_minimum_length_constraint
|
|
22
|
+
registry = Crawlscope::SchemaRegistry.new(schemas: {"Thing" => {type: "object", properties: {name: {type: "string", minLength: 3}}}})
|
|
23
|
+
error = registry.validate({"@type" => "Thing", "name" => "x"}).first
|
|
24
|
+
|
|
25
|
+
assert_equal "Thing.name: 1 character; minimum 3", format_error(error)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def test_formats_measured_headline_length_with_symbol_or_json_keys
|
|
29
|
+
error = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "é" * 111}).first
|
|
30
|
+
|
|
31
|
+
assert_equal "Article.headline: 111 characters; maximum 110", format_error(error)
|
|
32
|
+
assert_equal "Article.headline: 111 characters; maximum 110", format_error(JSON.parse(JSON.generate(error)))
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def test_formats_legacy_length_errors_without_a_measurement
|
|
36
|
+
error = {type: "Article", field: "headline", issue: "The property '#/headline' was not of a maximum string length of 110"}
|
|
37
|
+
|
|
38
|
+
assert_equal "Article.headline: exceeds 110 characters", format_error(error)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
def format_error(error)
|
|
44
|
+
Crawlscope::StructuredData::ErrorFormatter.format(error)
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -29,4 +29,30 @@ class CrawlscopeStructuredDataReporterTest < Minitest::Test
|
|
|
29
29
|
assert_includes output, "headline: is required"
|
|
30
30
|
assert_includes output, "/tmp/structured_data_report.json"
|
|
31
31
|
end
|
|
32
|
+
|
|
33
|
+
def test_formats_real_schema_errors_in_summary_and_detailed_reports
|
|
34
|
+
errors = Crawlscope::SchemaRegistry.default.validate({"@type" => "Article", "headline" => "x" * 111})
|
|
35
|
+
result = Crawlscope::StructuredData::Audit::Outcome.new(entries: [
|
|
36
|
+
Crawlscope::StructuredData::Audit::Page.new(
|
|
37
|
+
url: "https://example.com/article", status: 200,
|
|
38
|
+
structured_items: [{source: "json-ld", data: {"@type" => "Article"}}],
|
|
39
|
+
errors: [{type: "Article", source: "json-ld", errors: errors}],
|
|
40
|
+
fetch_error: nil, content_type: "text/html", skipped_reason: nil
|
|
41
|
+
)
|
|
42
|
+
])
|
|
43
|
+
io = StringIO.new
|
|
44
|
+
reporter = Crawlscope::StructuredData::Reporter.new(io: io)
|
|
45
|
+
|
|
46
|
+
reporter.report(result)
|
|
47
|
+
assert_includes io.string, "JSON-LD:\n - Article.headline: 111 characters; maximum 110"
|
|
48
|
+
refute_includes io.string, "in schema"
|
|
49
|
+
|
|
50
|
+
io.truncate(0)
|
|
51
|
+
io.rewind
|
|
52
|
+
reporter.details(result, debug: false, renderer: :http)
|
|
53
|
+
assert_includes io.string, "Validation: FAILED\n JSON-LD:\n - Article.headline: 111 characters; maximum 110"
|
|
54
|
+
refute_includes io.string, "=" * 80
|
|
55
|
+
refute_includes io.string, "in schema"
|
|
56
|
+
assert_includes errors.first[:issue], "in schema"
|
|
57
|
+
end
|
|
32
58
|
end
|
|
@@ -31,6 +31,8 @@ class CrawlscopeStructuredDataRuleTest < Minitest::Test
|
|
|
31
31
|
|
|
32
32
|
assert_equal [:structured_data_schema_error], issues.to_a.map(&:code)
|
|
33
33
|
assert_includes issues.to_a.first.message, "headline"
|
|
34
|
+
assert_equal "JSON-LD Article.headline: is required", issues.to_a.first.message
|
|
35
|
+
assert_includes issues.to_a.first.details[:errors].first[:issue], "in schema"
|
|
34
36
|
end
|
|
35
37
|
|
|
36
38
|
def test_reports_parse_errors_for_invalid_json_ld
|
|
@@ -44,6 +44,49 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
|
|
|
44
44
|
assert_operator issue.details[:similarity], :>=, issue.details[:threshold]
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
+
def test_does_not_report_redirect_aliases_as_duplicate_pages
|
|
48
|
+
pages = [
|
|
49
|
+
page(url: "https://example.com/old", final_url: "https://example.com/current"),
|
|
50
|
+
page(url: "https://example.com/alias", final_url: "https://example.com/current"),
|
|
51
|
+
page(url: "https://example.com/current")
|
|
52
|
+
]
|
|
53
|
+
issues = Crawlscope::IssueCollection.new
|
|
54
|
+
|
|
55
|
+
Crawlscope::Rules::Uniqueness.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
|
|
56
|
+
|
|
57
|
+
assert_empty issues.to_a
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def test_reports_direct_duplicate_pages_without_including_redirect_aliases
|
|
61
|
+
pages = [
|
|
62
|
+
page(url: "https://example.com/old", final_url: "https://example.com/a"),
|
|
63
|
+
page(url: "https://example.com/a"),
|
|
64
|
+
page(url: "https://example.com/b")
|
|
65
|
+
]
|
|
66
|
+
issues = Crawlscope::IssueCollection.new
|
|
67
|
+
|
|
68
|
+
Crawlscope::Rules::Uniqueness.new.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
|
|
69
|
+
|
|
70
|
+
assert_equal 4, issues.size
|
|
71
|
+
issues.each do |issue|
|
|
72
|
+
assert_equal ["https://example.com/a", "https://example.com/b"], issue.details[:urls]
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def test_excludes_redirects_from_near_duplicate_checks_and_scan_limit
|
|
77
|
+
pages = [
|
|
78
|
+
page(url: "https://example.com/old", final_url: "https://example.com/other", content: near_duplicate_content("reliable")),
|
|
79
|
+
page(url: "https://example.com/current", content: near_duplicate_content("dependable"))
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
[Crawlscope::Rules::Uniqueness.new, Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)].each do |rule|
|
|
83
|
+
issues = Crawlscope::IssueCollection.new
|
|
84
|
+
rule.call(urls: pages.map(&:url), pages: pages, issues: issues, context: {})
|
|
85
|
+
|
|
86
|
+
assert_empty issues.to_a
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
47
90
|
def test_skips_near_duplicate_scan_when_page_count_exceeds_limit
|
|
48
91
|
issues = Crawlscope::IssueCollection.new
|
|
49
92
|
rule = Crawlscope::Rules::Uniqueness.new(max_near_duplicate_pages: 1)
|
|
@@ -72,7 +115,7 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
|
|
|
72
115
|
TEXT
|
|
73
116
|
end
|
|
74
117
|
|
|
75
|
-
def page(url:, content: nil, canonical: nil)
|
|
118
|
+
def page(url:, content: nil, canonical: nil, final_url: url)
|
|
76
119
|
repeated_text = content || ("Useful content " * 30).strip
|
|
77
120
|
canonical_tag = canonical ? %(<link rel="canonical" href="#{canonical}">) : ""
|
|
78
121
|
body = <<~HTML
|
|
@@ -91,8 +134,8 @@ class CrawlscopeUniquenessRuleTest < Minitest::Test
|
|
|
91
134
|
Crawlscope::Page.new(
|
|
92
135
|
url: url,
|
|
93
136
|
normalized_url: url,
|
|
94
|
-
final_url:
|
|
95
|
-
normalized_final_url:
|
|
137
|
+
final_url: final_url,
|
|
138
|
+
normalized_final_url: final_url,
|
|
96
139
|
status: 200,
|
|
97
140
|
headers: {"content-type" => "text/html"},
|
|
98
141
|
body: body,
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlscope
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.8.
|
|
4
|
+
version: 0.8.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Paulo Fidalgo
|
|
@@ -257,6 +257,7 @@ files:
|
|
|
257
257
|
- lib/crawlscope/structured_data/audit.rb
|
|
258
258
|
- lib/crawlscope/structured_data/check.rb
|
|
259
259
|
- lib/crawlscope/structured_data/document.rb
|
|
260
|
+
- lib/crawlscope/structured_data/error_formatter.rb
|
|
260
261
|
- lib/crawlscope/structured_data/report.rb
|
|
261
262
|
- lib/crawlscope/structured_data/reporter.rb
|
|
262
263
|
- lib/crawlscope/structured_data/writer.rb
|
|
@@ -290,6 +291,7 @@ files:
|
|
|
290
291
|
- test/crawlscope/sitemap_test.rb
|
|
291
292
|
- test/crawlscope/structured_data_audit_test.rb
|
|
292
293
|
- test/crawlscope/structured_data_document_test.rb
|
|
294
|
+
- test/crawlscope/structured_data_error_formatter_test.rb
|
|
293
295
|
- test/crawlscope/structured_data_report_test.rb
|
|
294
296
|
- test/crawlscope/structured_data_reporter_test.rb
|
|
295
297
|
- test/crawlscope/structured_data_rule_test.rb
|
|
@@ -328,7 +330,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
328
330
|
- !ruby/object:Gem::Version
|
|
329
331
|
version: '0'
|
|
330
332
|
requirements: []
|
|
331
|
-
rubygems_version: 4.0.
|
|
333
|
+
rubygems_version: 4.0.20
|
|
332
334
|
specification_version: 4
|
|
333
335
|
summary: Audit sitemap URLs for metadata, structured data, uniqueness, and links
|
|
334
336
|
test_files: []
|