crawlscope 0.10.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/UPGRADE.md +76 -0
  4. data/docs/architecture.md +73 -0
  5. data/docs/rules.md +49 -0
  6. data/lib/crawlscope/configuration.rb +2 -1
  7. data/lib/crawlscope/crawl.rb +11 -79
  8. data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
  9. data/lib/crawlscope/crawling/target.rb +45 -0
  10. data/lib/crawlscope/crawling/target_resolver.rb +75 -0
  11. data/lib/crawlscope/documents/canonical.rb +22 -0
  12. data/lib/crawlscope/documents/robots.rb +57 -0
  13. data/lib/crawlscope/page.rb +4 -0
  14. data/lib/crawlscope/reporter.rb +6 -160
  15. data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
  16. data/lib/crawlscope/reporting/url_formatter.rb +29 -0
  17. data/lib/crawlscope/rule_registry.rb +2 -2
  18. data/lib/crawlscope/rules/content_quality.rb +18 -0
  19. data/lib/crawlscope/rules/indexability.rb +9 -14
  20. data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
  21. data/lib/crawlscope/rules/links/extractor.rb +100 -0
  22. data/lib/crawlscope/rules/links/graph.rb +39 -0
  23. data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
  24. data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
  25. data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
  26. data/lib/crawlscope/rules/links/scope.rb +23 -0
  27. data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
  28. data/lib/crawlscope/rules/links/url_checks.rb +54 -0
  29. data/lib/crawlscope/rules/links.rb +18 -600
  30. data/lib/crawlscope/rules/metadata.rb +2 -2
  31. data/lib/crawlscope/rules/structured_data.rb +26 -77
  32. data/lib/crawlscope/rules/uniqueness.rb +1 -1
  33. data/lib/crawlscope/schemas.rb +10 -2
  34. data/lib/crawlscope/server_timing/reporter.rb +2 -13
  35. data/lib/crawlscope/structured_data/audit.rb +17 -76
  36. data/lib/crawlscope/structured_data/check.rb +2 -0
  37. data/lib/crawlscope/structured_data/document.rb +14 -1
  38. data/lib/crawlscope/structured_data/entry.rb +31 -0
  39. data/lib/crawlscope/structured_data/outcome.rb +11 -0
  40. data/lib/crawlscope/structured_data/report.rb +6 -14
  41. data/lib/crawlscope/structured_data/reporter.rb +6 -11
  42. data/lib/crawlscope/structured_data/validator.rb +64 -0
  43. data/lib/crawlscope/version.rb +1 -1
  44. data/test/crawlscope/cli_test.rb +21 -0
  45. data/test/crawlscope/content_quality_rule_test.rb +51 -0
  46. data/test/crawlscope/links_rule_test.rb +16 -0
  47. data/test/crawlscope/reporter_test.rb +12 -0
  48. data/test/crawlscope/schema_registry_test.rb +22 -0
  49. data/test/crawlscope/structured_data_audit_test.rb +60 -0
  50. data/test/crawlscope/structured_data_report_test.rb +32 -0
  51. data/test/crawlscope/structured_data_rule_test.rb +11 -1
  52. data/test/crawlscope/structured_data_validator_test.rb +24 -0
  53. metadata +22 -1
@@ -5,6 +5,8 @@ require "uri"
5
5
  module Crawlscope
6
6
  module Rules
7
7
  class Links
8
+ LinkTarget = Crawling::Target
9
+
8
10
  LINK_SELECTORS = "a[href]"
9
11
  INTERNAL_PATH_PREFIXES_TO_SKIP = ["/rails/", "/cdn-cgi/"].freeze
10
12
  LINK_SCHEMES_TO_SKIP = ["mailto:", "tel:", "javascript:", "data:"].freeze
@@ -18,609 +20,25 @@ module Crawlscope
18
20
  end
19
21
 
20
22
  def call(urls:, pages:, issues:, context:)
21
- @allowed_statuses = context.fetch(:allowed_statuses)
22
- @base_url = context.fetch(:base_url)
23
- @concurrency = context_value(context, :concurrency, default: 1)
24
- @fetch_executor = context_value(context, :fetch_executor)
25
- @resolve_target = context.fetch(:resolve_target)
26
- @resolve_targets = context.resolve_targets if context.respond_to?(:resolve_targets)
27
- @resolve_targets ||= context[:resolve_targets] if context.respond_to?(:[])
28
- @base_host = URI.parse(@base_url).host
29
- @resolved_targets_by_url = {}
30
-
31
- links = extract_links(pages)
32
- validate_url_hygiene(urls, links, issues)
33
- resolved_links = resolve_links(links, issues)
34
- validate_nofollow_outgoing_links(resolved_links, issues)
35
- validate_http_internal_links(links, issues)
36
- validate_pages_with_no_outgoing_links(urls, pages, links, issues)
37
- validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
38
- validate_inbound_counts(urls, pages, resolved_links, issues)
39
- validate_canonical_targets(urls, pages, resolved_links, issues)
40
- end
41
-
42
- private
43
-
44
- def contextual_links(doc)
45
- doc.css(LINK_SELECTORS)
46
- end
47
-
48
- def extract_links(pages)
49
- html_pages = pages.select(&:html?)
50
- FetchExecutor.map(
51
- name: @fetch_executor,
52
- concurrency: @concurrency,
53
- items: html_pages
54
- ) { |page| page_links(page) }.flatten
55
- end
56
-
57
- def page_links(page)
58
- source_path = Url.path(page.normalized_url)
59
- return [] unless crawlable_source_path?(source_path)
60
-
61
- contextual_links(page.doc).filter_map do |node|
62
- link_for(page: page, source_path: source_path, node: node)
63
- end
64
- end
65
-
66
- def link_for(page:, source_path:, node:)
67
- href = node["href"].to_s.strip
68
- return unless crawlable_href?(href)
69
-
70
- anchor_text = normalize_anchor_text(node.text)
71
- return if anchor_text.empty?
72
-
73
- target_url = normalize_internal_link(page.normalized_url, href)
74
- return if target_url.nil?
75
-
76
- target_path = Url.path(target_url)
77
- return unless crawlable_path?(target_path)
78
-
79
- {
80
- anchor_text: anchor_text,
81
- http_internal_link: http_internal_link?(page.normalized_url, href),
82
- nofollow: nofollow_link?(node),
83
- source_path: source_path,
84
- source_url: page.normalized_url,
85
- target_path: target_path,
86
- target_url: target_url
87
- }
88
- end
89
-
90
- def crawlable_href?(href)
91
- return false if href.empty?
92
- return false if href.start_with?("#")
93
-
94
- LINK_SCHEMES_TO_SKIP.none? { |prefix| href.start_with?(prefix) }
95
- end
96
-
97
- def crawlable_path?(path)
98
- !path.nil? && !skip_internal_path?(path)
99
- end
100
-
101
- def normalize_anchor_text(text)
102
- text.to_s.gsub(/\s+/, " ").strip
103
- end
104
-
105
- def nofollow_link?(node)
106
- node["rel"].to_s.split(/\s+/).any? { |value| value.casecmp?("nofollow") }
107
- end
108
-
109
- def http_internal_link?(source_url, href)
110
- source_uri = URI.parse(source_url.to_s)
111
- target_uri = URI.parse(URI.join(source_url, href).to_s)
112
-
113
- source_uri.scheme == "https" && target_uri.scheme == "http" && target_uri.host == @base_host
114
- rescue URI::InvalidURIError
115
- false
116
- end
117
-
118
- def normalize_internal_link(source_url, href)
119
- absolute_url = URI.join(source_url, href).to_s
120
- uri = URI.parse(absolute_url)
121
- return if uri.host != @base_host
122
-
123
- uri.fragment = nil
124
- Url.normalize(uri.to_s, base_url: @base_url)
125
- rescue URI::InvalidURIError
126
- nil
127
- end
128
-
129
- def report_broken_target(target_url, grouped_links, issues, status)
130
- source_urls = grouped_links.map { |link| link[:source_url] }.uniq
131
- issues.add(
132
- code: :broken_internal_link,
133
- severity: :warning,
134
- category: :links,
135
- url: target_url,
136
- message: "linked URL returns HTTP #{status}",
137
- details: {source_urls: source_urls, status: status}
138
- )
139
- end
140
-
141
- def validate_nofollow_outgoing_links(links, issues)
142
- links.select do |link|
143
- link[:nofollow] && link[:source_url] != link[:final_url] && !target_for(link[:target_url]).noindex?
144
- end.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
145
- target_urls = grouped_links.map { |link| link[:target_url] }.uniq
146
-
147
- issues.add(
148
- code: :nofollow_internal_outlinks,
149
- severity: :warning,
150
- category: :links,
151
- url: source_url,
152
- message: "page has nofollow outgoing internal links",
153
- details: {target_urls: target_urls}
154
- )
155
- end
156
- end
157
-
158
- def validate_http_internal_links(links, issues)
159
- links.select { |link| link[:http_internal_link] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
160
- target_urls = grouped_links.map { |link| link[:target_url] }.uniq
161
-
162
- issues.add(
163
- code: :http_internal_link,
164
- severity: :warning,
165
- category: :links,
166
- url: source_url,
167
- message: "HTTPS page links to internal HTTP URL",
168
- details: {target_urls: target_urls}
169
- )
170
- end
171
- end
172
-
173
- def report_unresolved_target(target_url, grouped_links, issues, resolution)
174
- source_urls = grouped_links.map { |link| link[:source_url] }.uniq
175
- suffix = (resolution && resolution[:error]) ? ": #{resolution[:error]}" : ""
176
-
177
- issues.add(
178
- code: :unresolved_internal_link,
179
- severity: :warning,
180
- category: :links,
181
- url: target_url,
182
- message: "cannot check linked URL#{suffix}",
183
- details: {error: resolution && resolution[:error], source_urls: source_urls}
184
- )
185
- end
186
-
187
- def resolve_links(links, issues)
188
- resolved_links = []
189
- grouped_links_by_target = links.group_by { |link| link[:target_url] }
190
- targets_by_url = resolve_targets(grouped_links_by_target.keys)
191
- @resolved_targets_by_url.merge!(targets_by_url)
192
-
193
- grouped_links_by_target.each do |target_url, grouped_links|
194
- target = target_for(target_url)
195
-
196
- if target.unresolved?
197
- report_unresolved_target(target_url, grouped_links, issues, target.resolution)
198
- next
199
- end
200
-
201
- if target.ignored_error?
202
- next
203
- end
204
-
205
- unless target.allowed?(@allowed_statuses)
206
- report_broken_target(target_url, grouped_links, issues, target.status)
207
- next
208
- end
209
-
210
- report_redirect_target(target_url, grouped_links, issues, target) if target.redirect?
211
- next unless crawlable_path?(target.final_path)
212
-
213
- grouped_links.each do |link|
214
- resolved_links << link.merge(final_path: target.final_path, final_url: target.final_url)
215
- end
216
- end
217
-
218
- resolved_links
219
- end
220
-
221
- def report_redirect_target(target_url, grouped_links, issues, target)
222
- source_urls = grouped_links.map { |link| link[:source_url] }.uniq
223
- issues.add(
224
- code: :internal_link_redirects,
225
- severity: :warning,
226
- category: :links,
227
- url: target_url,
228
- message: "internal link redirects to #{target.final_url}",
229
- details: {final_url: target.final_url, source_urls: source_urls, status: target.status}
230
- )
231
- end
232
-
233
- def resolve_target(target_url)
234
- resolution = @resolve_target.call(target_url)
235
- LinkTarget.new(target_url: target_url, resolution: resolution)
236
- end
237
-
238
- def resolve_targets(target_urls)
239
- if @resolve_targets
240
- @resolve_targets.call(target_urls).to_h do |target_url, resolution|
241
- [target_url, LinkTarget.new(target_url: target_url, resolution: resolution)]
242
- end
243
- else
244
- target_urls.to_h { |target_url| [target_url, resolve_target(target_url)] }
245
- end
246
- end
247
-
248
- def target_for(target_url)
249
- @resolved_targets_by_url.fetch(target_url) do
250
- target = resolve_target(target_url)
251
- @resolved_targets_by_url[target_url] = target
252
- end
253
- end
254
-
255
- LinkTarget = Data.define(:target_url, :resolution) do
256
- def allowed?(statuses)
257
- statuses.include?(status)
258
- end
259
-
260
- def final_path
261
- Url.path(final_url)
262
- end
263
-
264
- def final_url
265
- value = resolution[:final_url].to_s
266
- value.empty? ? target_url : value
267
- end
268
-
269
- def ignored_error?
270
- resolution && status.nil? && resolution[:crawled] && resolution[:error]
271
- end
272
-
273
- def html?
274
- resolution && resolution[:html]
275
- end
276
-
277
- def noindex?
278
- Crawlscope::Rules::Indexability.noindex_header?(resolution[:headers] || {}) ||
279
- Crawlscope::Rules::Indexability.noindex_meta?(resolution[:doc])
280
- end
281
-
282
- def status
283
- resolution && resolution[:status]
284
- end
285
-
286
- def redirect?
287
- (status && (300..399).cover?(status.to_i)) || final_url != target_url
288
- end
289
-
290
- def unresolved?
291
- resolution.nil? || (status.nil? && !ignored_error?)
292
- end
293
- end
294
-
295
- def crawlable_source_path?(path)
296
- !path.nil? && INTERNAL_PATH_PREFIXES_TO_SKIP.none? { |prefix| path.start_with?(prefix) }
297
- end
298
-
299
- def skip_internal_path?(path)
300
- return true if path == "/"
301
-
302
- INTERNAL_PATH_PREFIXES_TO_SKIP.any? { |prefix| path.start_with?(prefix) }
303
- end
304
-
305
- def validate_inbound_counts(urls, pages, resolved_links, issues)
306
- sitemap_urls = urls.filter_map do |url|
307
- normalized_url = Url.normalize(url, base_url: @base_url)
308
- path = Url.path(normalized_url)
309
- next if path.nil?
310
- next if skip_internal_path?(path)
311
-
312
- normalized_url
313
- end.to_set
314
- return if sitemap_urls.size < 2
315
-
316
- html_urls = pages.each_with_object(Set.new) do |page, result|
317
- next unless page.html?
318
-
319
- [page.normalized_url, page.normalized_final_url].compact.each do |url|
320
- path = Url.path(url)
321
- next if path.nil?
322
- next if skip_internal_path?(path)
323
-
324
- result << url
325
- end
326
- end
327
-
328
- inbound_anchor_counts = Hash.new(0)
329
- dofollow_inbound_counts = Hash.new(0)
330
- nofollow_inbound_counts = Hash.new(0)
331
- sample_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
332
- dofollow_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
333
- nofollow_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
334
-
335
- resolved_links.each do |link|
336
- target_url = link[:final_url]
337
- next unless sitemap_urls.include?(target_url)
338
- next if target_for(link[:target_url]).noindex?
339
- next if link[:source_url] == target_url
340
-
341
- inbound_anchor_counts[target_url] += 1
342
- source_samples = sample_sources_by_target[target_url]
343
- source_samples << link[:source_url] unless source_samples.include?(link[:source_url])
344
-
345
- if link[:nofollow]
346
- nofollow_inbound_counts[target_url] += 1
347
- nofollow_sources = nofollow_sources_by_target[target_url]
348
- nofollow_sources << link[:source_url] unless nofollow_sources.include?(link[:source_url])
349
- else
350
- dofollow_inbound_counts[target_url] += 1
351
- dofollow_sources = dofollow_sources_by_target[target_url]
352
- dofollow_sources << link[:source_url] unless dofollow_sources.include?(link[:source_url])
353
- end
354
- end
355
-
356
- sitemap_urls.each do |target_url|
357
- next unless html_urls.include?(target_url)
358
-
359
- inbound_count = inbound_anchor_counts[target_url]
360
- dofollow_count = dofollow_inbound_counts[target_url]
361
- nofollow_count = nofollow_inbound_counts[target_url]
362
-
363
- report_orphan_page(target_url, issues) if inbound_count.zero?
364
-
365
- if inbound_count.positive? && inbound_count < MIN_INBOUND_ANCHOR_LINKS
366
- source_samples = sample_sources_by_target[target_url]
367
-
368
- issues.add(
369
- code: :low_inbound_anchor_links,
370
- severity: :warning,
371
- category: :links,
372
- url: target_url,
373
- message: "incoming internal links: #{inbound_count}; minimum #{MIN_INBOUND_ANCHOR_LINKS}",
374
- details: {inbound_count: inbound_count, minimum: MIN_INBOUND_ANCHOR_LINKS, source_urls: source_samples}
375
- )
376
- end
377
-
378
- report_low_dofollow_inlinks(target_url, dofollow_count, dofollow_sources_by_target, issues)
379
- report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[target_url], issues)
380
- report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[target_url], dofollow_sources_by_target[target_url], issues)
381
- end
382
- end
383
-
384
- def validate_url_hygiene(urls, links, issues)
385
- checked_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }
386
- checked_urls.concat(links.map { |link| link[:target_url] })
387
-
388
- checked_urls.compact.uniq.each do |url|
389
- report_url_double_slash(url, issues)
390
- report_url_too_long(url, issues)
391
- end
392
- end
393
-
394
- def report_url_double_slash(url, issues)
395
- path = URI.parse(url).path.to_s
396
- return unless path.match?(%r{//+})
397
-
398
- issues.add(
399
- code: :url_double_slash,
400
- severity: :notice,
401
- category: :url,
402
- url: url,
403
- message: "URL path contains duplicate slashes",
404
- details: {path: path}
405
- )
406
- rescue URI::InvalidURIError
407
- nil
408
- end
409
-
410
- def report_url_too_long(url, issues)
411
- return unless url.length > 2_048
412
-
413
- issues.add(
414
- code: :url_too_long,
415
- severity: :notice,
416
- category: :url,
417
- url: url,
418
- message: "URL: #{url.length} characters; maximum 2048",
419
- details: {length: url.length, maximum: 2_048}
420
- )
421
- end
422
-
423
- def validate_pages_with_no_outgoing_links(urls, pages, links, issues)
424
- sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
425
- return if sitemap_urls.size < 2
426
-
427
- source_paths_with_links = links.map { |link| link[:source_path] }.to_set
428
-
429
- pages.each do |page|
430
- next unless page.html?
431
- next unless sitemap_urls.include?(page.normalized_url)
432
-
433
- source_path = Url.path(page.normalized_url)
434
- next unless crawlable_source_path?(source_path)
435
- next if source_paths_with_links.include?(source_path)
436
-
437
- issues.add(
438
- code: :page_has_no_outgoing_links,
439
- severity: :warning,
440
- category: :links,
441
- url: page.url,
442
- message: "page has no outgoing internal links",
443
- details: {}
444
- )
445
- end
446
- end
447
-
448
- def validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
449
- sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
450
- resolved_links.group_by { |link| link[:final_url] }.each do |final_url, grouped_links|
451
- link = grouped_links.first
452
- next if sitemap_urls.include?(final_url)
453
- next unless crawlable_path?(link[:final_path])
454
-
455
- target = target_for(final_url)
456
- next unless target.allowed?(@allowed_statuses) && target.html?
457
- next if target.noindex?
458
-
459
- issues.add(
460
- code: :indexable_page_missing_from_sitemap,
461
- severity: :warning,
462
- category: :sitemaps,
463
- url: final_url,
464
- message: "linked page is missing from sitemap",
465
- details: {source_url: link[:source_url], source_urls: grouped_links.map { |entry| entry[:source_url] }.uniq}
23
+ scope = Scope.new
24
+ targets = TargetLookup.new(context: context)
25
+ links = Extractor.new(base_url: context.fetch(:base_url),
26
+ concurrency: context_value(context, :concurrency, default: 1),
27
+ fetch_executor: context_value(context, :fetch_executor), scope: scope).call(pages)
28
+ UrlChecks.new(base_url: context.fetch(:base_url)).call(urls: urls, links: links, issues: issues)
29
+ resolved_links = ResolutionCheck.new(targets: targets,
30
+ allowed_statuses: context.fetch(:allowed_statuses), scope: scope).call(links, issues: issues)
31
+ graph = Graph.new(links: resolved_links, targets: targets)
32
+ OutgoingChecks.new(base_url: context.fetch(:base_url), targets: targets,
33
+ allowed_statuses: context.fetch(:allowed_statuses), scope: scope).call(
34
+ urls: urls, pages: pages, links: links, resolved_links: resolved_links, issues: issues
466
35
  )
467
- end
468
- end
469
-
470
- def report_orphan_page(target_url, issues)
471
- issues.add(
472
- code: :orphan_page,
473
- severity: :warning,
474
- category: :links,
475
- url: target_url,
476
- message: "page has no incoming internal links",
477
- details: {}
478
- )
479
- end
480
-
481
- def report_low_dofollow_inlinks(target_url, dofollow_count, sources_by_target, issues)
482
- return if dofollow_count.zero?
483
- return if dofollow_count >= MIN_DOFOLLOW_INBOUND_LINKS
484
-
485
- source_samples = sources_by_target[target_url]
486
-
487
- issues.add(
488
- code: :low_dofollow_inlinks,
489
- severity: :warning,
490
- category: :links,
491
- url: target_url,
492
- message: "incoming internal links without nofollow: #{dofollow_count}; minimum #{MIN_DOFOLLOW_INBOUND_LINKS}",
493
- details: {dofollow_inbound_count: dofollow_count, minimum: MIN_DOFOLLOW_INBOUND_LINKS, source_urls: source_samples}
494
- )
495
- end
496
-
497
- def report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, issues)
498
- return unless nofollow_count.positive? && dofollow_count.zero?
499
-
500
- issues.add(
501
- code: :only_nofollow_internal_inlinks,
502
- severity: :warning,
503
- category: :links,
504
- url: target_url,
505
- message: "all incoming internal links use nofollow",
506
- details: {nofollow_inbound_count: nofollow_count, source_urls: nofollow_sources}
507
- )
508
- end
509
-
510
- def report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, dofollow_sources, issues)
511
- return unless nofollow_count.positive? && dofollow_count.positive?
512
-
513
- issues.add(
514
- code: :mixed_follow_internal_inlinks,
515
- severity: :notice,
516
- category: :links,
517
- url: target_url,
518
- message: "page has nofollow and dofollow incoming internal links",
519
- details: {
520
- dofollow_inbound_count: dofollow_count,
521
- nofollow_inbound_count: nofollow_count,
522
- dofollow_source_urls: dofollow_sources,
523
- nofollow_source_urls: nofollow_sources
524
- }
525
- )
526
- end
527
-
528
- def validate_canonical_targets(urls, pages, resolved_links, issues)
529
- sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact
530
- sitemap_pages = pages.select do |page|
531
- page.html? && sitemap_urls.include?(page.normalized_url) && page.normalized_url.to_s == page.normalized_final_url.to_s
532
- end
533
- return if sitemap_pages.size < 2
534
-
535
- dofollow_counts_by_url = dofollow_counts_by_final_url(resolved_links)
536
- canonical_entries = canonical_entries_for(sitemap_pages)
537
- @resolved_targets_by_url.merge!(resolve_targets(canonical_entries.map { |entry| entry.fetch(:canonical_url) }))
538
-
539
- canonical_entries.each do |entry|
540
- page = entry.fetch(:page)
541
- canonical_url = entry.fetch(:canonical_url)
542
- canonical_path = entry.fetch(:canonical_path)
543
- if canonical_path && !root_path?(canonical_path) && dofollow_counts_by_url[canonical_url].zero?
544
- issues.add(
545
- code: :canonical_no_internal_inlinks,
546
- severity: :warning,
547
- category: :links,
548
- url: canonical_url,
549
- message: "canonical URL has no incoming internal links without nofollow",
550
- details: {source_url: page.url}
551
- )
552
- end
553
-
554
- validate_canonical_target_status(page, canonical_url, issues)
555
- end
36
+ InboundChecks.new(base_url: context.fetch(:base_url), graph: graph, scope: scope).call(urls: urls, pages: pages, issues: issues)
37
+ CanonicalChecks.new(base_url: context.fetch(:base_url), targets: targets,
38
+ allowed_statuses: context.fetch(:allowed_statuses), graph: graph).call(urls: urls, pages: pages, issues: issues)
556
39
  end
557
40
 
558
- def canonical_entries_for(pages)
559
- pages.filter_map do |page|
560
- canonical_url = canonical_url_for(page)
561
- next if canonical_url.nil?
562
-
563
- target_uri = URI.parse(canonical_url)
564
- next if target_uri.host != @base_host
565
-
566
- {
567
- canonical_path: Url.path(canonical_url),
568
- canonical_url: canonical_url,
569
- page: page
570
- }
571
- rescue URI::InvalidURIError
572
- nil
573
- end
574
- end
575
-
576
- def dofollow_counts_by_final_url(resolved_links)
577
- resolved_links.each_with_object(Hash.new(0)) do |link, counts|
578
- next if link[:nofollow]
579
- next if link[:source_url] == link[:final_url]
580
- next if target_for(link[:target_url]).noindex?
581
-
582
- counts[link[:final_url]] += 1
583
- end
584
- end
585
-
586
- def canonical_url_for(page)
587
- canonical = page.doc.at_css('link[rel="canonical"]')&.[]("href").to_s.strip
588
- return if canonical.empty?
589
-
590
- Url.normalize(canonical, base_url: page.url)
591
- end
592
-
593
- def root_path?(path)
594
- path == "/"
595
- end
596
-
597
- def validate_canonical_target_status(page, canonical_url, issues)
598
- target = target_for(canonical_url)
599
-
600
- if target.unresolved? || target.ignored_error?
601
- return
602
- end
603
-
604
- if target.redirect?
605
- issues.add(
606
- code: :canonical_points_to_redirect,
607
- severity: :warning,
608
- category: :metadata,
609
- url: page.url,
610
- message: "canonical #{canonical_url} redirects to #{target.final_url}",
611
- details: {canonical: canonical_url, final_url: target.final_url, status: target.status}
612
- )
613
- elsif !target.allowed?(@allowed_statuses)
614
- issues.add(
615
- code: :canonical_points_to_error,
616
- severity: :warning,
617
- category: :metadata,
618
- url: page.url,
619
- message: "canonical #{canonical_url} returns HTTP #{target.status}",
620
- details: {canonical: canonical_url, status: target.status}
621
- )
622
- end
623
- end
41
+ private
624
42
 
625
43
  def context_value(context, name, default: nil)
626
44
  if context.respond_to?(name)
@@ -124,14 +124,14 @@ module Crawlscope
124
124
  end
125
125
 
126
126
  def validate_canonical(page, issues, sitemap_urls)
127
- canonical = page.doc.at_css('link[rel="canonical"]')&.[]("href").to_s.strip
127
+ canonical = page.canonical.href
128
128
 
129
129
  if canonical.empty?
130
130
  issues.add(code: :missing_canonical, severity: :warning, category: :metadata, url: page.url, message: "missing canonical link", details: {})
131
131
  return
132
132
  end
133
133
 
134
- normalized_canonical = Url.normalize(canonical, base_url: page.url)
134
+ normalized_canonical = page.canonical.url
135
135
  normalized_page_url = Url.normalize(page.url, base_url: page.url)
136
136
  return if canonical_matches_page?(normalized_canonical, normalized_page_url)
137
137