crawlscope 0.10.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/UPGRADE.md +76 -0
  4. data/docs/architecture.md +73 -0
  5. data/docs/rules.md +49 -0
  6. data/lib/crawlscope/configuration.rb +2 -1
  7. data/lib/crawlscope/crawl.rb +11 -79
  8. data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
  9. data/lib/crawlscope/crawling/target.rb +45 -0
  10. data/lib/crawlscope/crawling/target_resolver.rb +75 -0
  11. data/lib/crawlscope/documents/canonical.rb +22 -0
  12. data/lib/crawlscope/documents/robots.rb +57 -0
  13. data/lib/crawlscope/page.rb +4 -0
  14. data/lib/crawlscope/reporter.rb +6 -160
  15. data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
  16. data/lib/crawlscope/reporting/url_formatter.rb +29 -0
  17. data/lib/crawlscope/rule_registry.rb +2 -2
  18. data/lib/crawlscope/rules/content_quality.rb +18 -0
  19. data/lib/crawlscope/rules/indexability.rb +9 -14
  20. data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
  21. data/lib/crawlscope/rules/links/extractor.rb +100 -0
  22. data/lib/crawlscope/rules/links/graph.rb +39 -0
  23. data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
  24. data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
  25. data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
  26. data/lib/crawlscope/rules/links/scope.rb +23 -0
  27. data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
  28. data/lib/crawlscope/rules/links/url_checks.rb +54 -0
  29. data/lib/crawlscope/rules/links.rb +18 -600
  30. data/lib/crawlscope/rules/metadata.rb +2 -2
  31. data/lib/crawlscope/rules/structured_data.rb +26 -77
  32. data/lib/crawlscope/rules/uniqueness.rb +1 -1
  33. data/lib/crawlscope/schemas.rb +10 -2
  34. data/lib/crawlscope/server_timing/reporter.rb +2 -13
  35. data/lib/crawlscope/structured_data/audit.rb +17 -76
  36. data/lib/crawlscope/structured_data/check.rb +2 -0
  37. data/lib/crawlscope/structured_data/document.rb +14 -1
  38. data/lib/crawlscope/structured_data/entry.rb +31 -0
  39. data/lib/crawlscope/structured_data/outcome.rb +11 -0
  40. data/lib/crawlscope/structured_data/report.rb +6 -14
  41. data/lib/crawlscope/structured_data/reporter.rb +6 -11
  42. data/lib/crawlscope/structured_data/validator.rb +64 -0
  43. data/lib/crawlscope/version.rb +1 -1
  44. data/test/crawlscope/cli_test.rb +21 -0
  45. data/test/crawlscope/content_quality_rule_test.rb +51 -0
  46. data/test/crawlscope/links_rule_test.rb +16 -0
  47. data/test/crawlscope/reporter_test.rb +12 -0
  48. data/test/crawlscope/schema_registry_test.rb +22 -0
  49. data/test/crawlscope/structured_data_audit_test.rb +60 -0
  50. data/test/crawlscope/structured_data_report_test.rb +32 -0
  51. data/test/crawlscope/structured_data_rule_test.rb +11 -1
  52. data/test/crawlscope/structured_data_validator_test.rb +24 -0
  53. metadata +22 -1
@@ -1,40 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "uri"
4
-
5
3
  module Crawlscope
6
4
  class Reporter
7
- GROUP_LABELS = {
8
- structured_data_schema_error: "Schema errors",
9
- structured_data_parse_error: "Parse errors",
10
- structured_data_missing_type: "Missing @type",
11
- sitemap_redirect_url: "Redirected URLs",
12
- non_canonical_page_in_sitemap: "URLs with a different canonical",
13
- indexable_page_missing_from_sitemap: "Linked pages missing from sitemap",
14
- sitemap_noindex_url: "URLs that block indexing",
15
- low_dofollow_inlinks: "Too few incoming links without nofollow",
16
- low_inbound_anchor_links: "Too few incoming links",
17
- low_unique_token_ratio: "Repeated words",
18
- low_visible_text_ratio: "Too little text for the HTML size",
19
- thin_visible_text: "Too few words",
20
- orphan_page: "Pages with no incoming links",
21
- nofollow_internal_outlinks: "Outgoing links with nofollow",
22
- only_nofollow_internal_inlinks: "Incoming links all use nofollow",
23
- mixed_follow_internal_inlinks: "Incoming links with and without nofollow",
24
- http_internal_link: "Links from HTTPS to HTTP",
25
- canonical_no_internal_inlinks: "Canonical URLs with no incoming links without nofollow",
26
- canonical_points_to_error: "Canonical URLs that return errors",
27
- url_double_slash: "Repeated slashes in URL paths",
28
- multiple_h1: "Multiple <h1> tags",
29
- multiple_title_tags: "Multiple <title> tags",
30
- incomplete_open_graph_tags: "Missing Open Graph tags",
31
- title_too_long: "Title exceeds #{Rules::Metadata::TITLE_MAX_LENGTH} characters",
32
- meta_description_too_long: "Meta description exceeds #{Rules::Metadata::DESCRIPTION_MAX_LENGTH} characters",
33
- meta_description_too_short: "Meta description below #{Rules::Metadata::DESCRIPTION_MIN_LENGTH} characters",
34
- duplicate_content_fingerprint: "Duplicate page text",
35
- near_duplicate_content: "Similar page text",
36
- near_duplicate_scan_skipped: "Similar-page check skipped"
37
- }.freeze
5
+ GROUP_LABELS = Reporting::IssuePresenter::GROUP_LABELS
38
6
 
39
7
  def initialize(io:)
40
8
  @io = io
@@ -86,21 +54,14 @@ module Crawlscope
86
54
  grouped
87
55
  .sort_by { |(category, code), grouped_issues| [grouped_issues.map { |issue| severity_order(issue.severity) }.min, -grouped_issues.size, category.to_s, code.to_s] }
88
56
  .each do |(category, code), grouped_issues|
89
- label = GROUP_LABELS.fetch(code) { code.to_s.tr("_", " ").capitalize }
57
+ label = Reporting::IssuePresenter.label(code)
90
58
  @io.puts("#{category.to_s.tr("_", " ").capitalize}: #{label} (#{severity_summary_for(grouped_issues)})")
91
59
 
60
+ urls = Reporting::UrlFormatter.new(base_url: base_url)
92
61
  grouped_issues.each do |issue|
93
- details = issue.details || {}
94
- if issue.code == :structured_data_schema_error && details[:errors]&.any?
95
- source = (details[:source] == "json-ld") ? "JSON-LD" : details[:source]
96
- @io.puts(" - #{relative_url(issue.url, base_url: base_url)} #{source}")
97
- details[:errors].each do |error|
98
- @io.puts(" #{StructuredData::ErrorFormatter.format(error)}")
99
- end
100
- else
101
- @io.puts(" - #{compact_issue(issue, base_url: base_url)}")
102
- report_related_urls(issue, details, base_url: base_url)
103
- end
62
+ first, *related = Reporting::IssuePresenter.new(issue, urls: urls).lines
63
+ @io.puts(" - #{first}")
64
+ related.each { |line| @io.puts(" #{line}") }
104
65
  end
105
66
 
106
67
  @io.puts("")
@@ -116,121 +77,6 @@ module Crawlscope
116
77
  .map { |severity, entries| "#{entries.size} #{pluralize(severity, entries.size)}" }.join(", ")
117
78
  end
118
79
 
119
- def compact_issue(issue, base_url:)
120
- parts = []
121
- parts << relative_url(issue.url, base_url: base_url) if issue.url
122
-
123
- detail = compact_detail(issue, base_url: base_url)
124
- parts << detail unless detail.empty?
125
-
126
- parts.compact.join(" ")
127
- end
128
-
129
- def compact_detail(issue, base_url:)
130
- details = issue.details || {}
131
- case issue.code
132
- when :redirected_page, :sitemap_redirect_url, :internal_link_redirects
133
- if details[:final_url]
134
- prefix = (issue.code == :sitemap_redirect_url) ? "sitemap URL redirects to" : "redirects to"
135
- status = details[:status] ? " (final HTTP #{details[:status]})" : ""
136
- return "#{prefix} #{relative_url(details[:final_url], base_url: base_url)}#{status}"
137
- end
138
- when :canonical_mismatch, :non_canonical_page_in_sitemap
139
- if details[:canonical]
140
- prefix = (issue.code == :non_canonical_page_in_sitemap) ? "sitemap lists this URL, but its canonical points to" : "canonical points to a different URL:"
141
- return "#{prefix} #{relative_url(details[:canonical], base_url: base_url)}"
142
- end
143
- when :unexpected_status
144
- return "HTTP #{details[:status]}" if details[:status]
145
- when :title_too_long, :meta_description_too_long, :meta_description_too_short
146
- return "#{details[:length]} characters" if details[:length]
147
- when :broken_internal_link
148
- return "linked URL returns HTTP #{details[:status]}"
149
- when :unresolved_internal_link
150
- return ["cannot check linked URL", details[:error]].compact.join(": ")
151
- when :indexable_page_missing_from_sitemap
152
- return "linked page is missing from sitemap"
153
- when :canonical_points_to_redirect
154
- status = details[:status] ? " (final HTTP #{details[:status]})" : ""
155
- return "canonical #{relative_url(details[:canonical], base_url: base_url)} redirects to #{relative_url(details[:final_url], base_url: base_url)}#{status}"
156
- when :canonical_points_to_error
157
- return "canonical #{relative_url(details[:canonical], base_url: base_url)} returns HTTP #{details[:status]}"
158
- when :low_dofollow_inlinks, :low_inbound_anchor_links
159
- count = details[:dofollow_inbound_count] || details[:inbound_count]
160
- label = (issue.code == :low_dofollow_inlinks) ? "incoming links without nofollow" : "incoming links"
161
- return "#{label}: #{count}; minimum #{details[:minimum]}"
162
- when :low_unique_token_ratio, :low_visible_text_ratio
163
- label = (issue.code == :low_unique_token_ratio) ? "unique words" : "visible text / HTML bytes"
164
- return "#{label}: #{percentage(details[:ratio])}; minimum #{percentage(details[:threshold])}"
165
- when :thin_visible_text
166
- return "visible words: #{details[:word_count]}; minimum #{details[:minimum]}"
167
- when :mixed_follow_internal_inlinks
168
- return "incoming links: #{details[:dofollow_inbound_count]} without nofollow, #{details[:nofollow_inbound_count]} with nofollow"
169
- when :only_nofollow_internal_inlinks
170
- return "all #{details[:nofollow_inbound_count]} incoming internal links use nofollow"
171
- when :duplicate_title, :duplicate_meta_description
172
- label = (issue.code == :duplicate_title) ? "same title" : "same meta description"
173
- return "#{label}: #{details[:value].to_s.inspect}"
174
- when :duplicate_content_fingerprint
175
- return "same page text"
176
- when :duplicate_pages_without_canonical
177
- return "same page text; at least one page has no canonical link"
178
- when :near_duplicate_content
179
- return "text similarity: #{percentage(details[:similarity])}; threshold #{percentage(details[:threshold])}"
180
- when :near_duplicate_scan_skipped
181
- return "similar-page check skipped: #{details[:page_count]} pages; limit #{details[:max_pages]}"
182
- when :structured_data_missing_type
183
- return "#{issue.message} at #{Array(details[:paths]).join(", ")}" if details[:paths]&.any?
184
- when :noindex_meta, :nofollow_meta, :noindex_follow_meta, :noindex_nofollow_meta,
185
- :noindex_header, :nofollow_header, :noindex_follow_header, :noindex_nofollow_header,
186
- :sitemap_noindex_url
187
- if details[:content]
188
- source = details[:name] || details[:source]
189
- label = source ? "#{source}: " : ""
190
- return "#{issue.message} (#{label}#{details[:content]})"
191
- end
192
- end
193
- issue.message
194
- end
195
-
196
- def report_related_urls(issue, details, base_url:)
197
- source_label = (issue.code == :canonical_no_internal_inlinks) ? "canonical on" : "linked from"
198
- sources = Array(details[:source_urls])
199
- sources += [details[:source_url]] if details[:source_url]
200
- relationships = {
201
- source_label => sources.uniq,
202
- "links to" => details[:target_urls],
203
- "page" => details[:urls],
204
- "linked from (without nofollow)" => details[:dofollow_source_urls],
205
- "linked from (nofollow)" => details[:nofollow_source_urls]
206
- }
207
- relationships.each do |label, urls|
208
- Array(urls).each do |url|
209
- @io.puts(" #{label}: #{relative_url(url, base_url: base_url)}")
210
- end
211
- end
212
- end
213
-
214
- def relative_url(url, base_url:)
215
- return url unless url && base_url
216
-
217
- uri = URI.parse(url)
218
- base_uri = URI.parse(base_url)
219
-
220
- return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
221
-
222
- relative = uri.path.to_s.empty? ? "/" : uri.path
223
- relative += "?#{uri.query}" if uri.query
224
- relative += "##{uri.fragment}" if uri.fragment
225
- relative
226
- rescue URI::InvalidURIError
227
- url
228
- end
229
-
230
- def percentage(value)
231
- "#{format("%.3f", value * 100).sub(/0+\z/, "").sub(/\.\z/, "")}%"
232
- end
233
-
234
80
  def pluralize(word, count)
235
81
  return word.to_s if count == 1
236
82
 
@@ -0,0 +1,162 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Reporting
5
+ class IssuePresenter
6
+ GROUP_LABELS = {
7
+ structured_data_schema_error: "Schema errors",
8
+ structured_data_parse_error: "Parse errors",
9
+ structured_data_missing_type: "Missing @type",
10
+ sitemap_redirect_url: "Redirected URLs",
11
+ non_canonical_page_in_sitemap: "URLs with a different canonical",
12
+ indexable_page_missing_from_sitemap: "Linked pages missing from sitemap",
13
+ sitemap_noindex_url: "URLs that block indexing",
14
+ low_dofollow_inlinks: "Too few incoming links without nofollow",
15
+ low_inbound_anchor_links: "Too few incoming links",
16
+ low_unique_token_ratio: "Repeated words",
17
+ low_visible_text_ratio: "Too little text for the HTML size",
18
+ thin_visible_text: "Too few words",
19
+ missing_image_alt: "Images missing alt attributes",
20
+ orphan_page: "Pages with no incoming links",
21
+ nofollow_internal_outlinks: "Outgoing links with nofollow",
22
+ only_nofollow_internal_inlinks: "Incoming links all use nofollow",
23
+ mixed_follow_internal_inlinks: "Incoming links with and without nofollow",
24
+ http_internal_link: "Links from HTTPS to HTTP",
25
+ canonical_no_internal_inlinks: "Canonical URLs with no incoming links without nofollow",
26
+ canonical_points_to_error: "Canonical URLs that return errors",
27
+ url_double_slash: "Repeated slashes in URL paths",
28
+ multiple_h1: "Multiple <h1> tags",
29
+ multiple_title_tags: "Multiple <title> tags",
30
+ incomplete_open_graph_tags: "Missing Open Graph tags",
31
+ title_too_long: "Title exceeds #{Rules::Metadata::TITLE_MAX_LENGTH} characters",
32
+ meta_description_too_long: "Meta description exceeds #{Rules::Metadata::DESCRIPTION_MAX_LENGTH} characters",
33
+ meta_description_too_short: "Meta description below #{Rules::Metadata::DESCRIPTION_MIN_LENGTH} characters",
34
+ duplicate_content_fingerprint: "Duplicate page text",
35
+ near_duplicate_content: "Similar page text",
36
+ near_duplicate_scan_skipped: "Similar-page check skipped"
37
+ }.freeze
38
+
39
+ def initialize(issue, urls:)
40
+ @issue = issue
41
+ @urls = urls
42
+ end
43
+
44
+ def self.label(code)
45
+ GROUP_LABELS.fetch(code) { code.to_s.tr("_", " ").capitalize }
46
+ end
47
+
48
+ def lines
49
+ details = issue.details || {}
50
+ if issue.code == :structured_data_schema_error && details[:errors]&.any?
51
+ source = (details[:source] == "json-ld") ? "JSON-LD" : details[:source]
52
+ ["#{@urls.call(issue.url)} #{source}", *details[:errors].map { |error| StructuredData::ErrorFormatter.format(error) }]
53
+ else
54
+ [summary, *related_lines]
55
+ end
56
+ end
57
+
58
+ private
59
+
60
+ attr_reader :issue
61
+
62
+ def summary
63
+ parts = []
64
+ parts << @urls.call(issue.url) if issue.url
65
+
66
+ text = detail
67
+ parts << text unless text.empty?
68
+
69
+ parts.compact.join(" ")
70
+ end
71
+
72
+ def detail
73
+ details = issue.details || {}
74
+ case issue.code
75
+ when :redirected_page, :sitemap_redirect_url, :internal_link_redirects
76
+ if details[:final_url]
77
+ prefix = (issue.code == :sitemap_redirect_url) ? "sitemap URL redirects to" : "redirects to"
78
+ status = details[:status] ? " (final HTTP #{details[:status]})" : ""
79
+ return "#{prefix} #{@urls.call(details[:final_url])}#{status}"
80
+ end
81
+ when :canonical_mismatch, :non_canonical_page_in_sitemap
82
+ if details[:canonical]
83
+ prefix = (issue.code == :non_canonical_page_in_sitemap) ? "sitemap lists this URL, but its canonical points to" : "canonical points to a different URL:"
84
+ return "#{prefix} #{@urls.call(details[:canonical])}"
85
+ end
86
+ when :unexpected_status
87
+ return "HTTP #{details[:status]}" if details[:status]
88
+ when :title_too_long, :meta_description_too_long, :meta_description_too_short
89
+ return "#{details[:length]} characters" if details[:length]
90
+ when :broken_internal_link
91
+ return "linked URL returns HTTP #{details[:status]}"
92
+ when :unresolved_internal_link
93
+ return ["cannot check linked URL", details[:error]].compact.join(": ")
94
+ when :indexable_page_missing_from_sitemap
95
+ return "linked page is missing from sitemap"
96
+ when :canonical_points_to_redirect
97
+ status = details[:status] ? " (final HTTP #{details[:status]})" : ""
98
+ return "canonical #{@urls.call(details[:canonical])} redirects to #{@urls.call(details[:final_url])}#{status}"
99
+ when :canonical_points_to_error
100
+ return "canonical #{@urls.call(details[:canonical])} returns HTTP #{details[:status]}"
101
+ when :low_dofollow_inlinks, :low_inbound_anchor_links
102
+ count = details[:dofollow_inbound_count] || details[:inbound_count]
103
+ label = (issue.code == :low_dofollow_inlinks) ? "incoming links without nofollow" : "incoming links"
104
+ return "#{label}: #{count}; minimum #{details[:minimum]}"
105
+ when :low_unique_token_ratio, :low_visible_text_ratio
106
+ label = (issue.code == :low_unique_token_ratio) ? "unique words" : "visible text / HTML bytes"
107
+ return "#{label}: #{percentage(details[:ratio])}; minimum #{percentage(details[:threshold])}"
108
+ when :thin_visible_text
109
+ return "visible words: #{details[:word_count]}; minimum #{details[:minimum]}"
110
+ when :mixed_follow_internal_inlinks
111
+ return "incoming links: #{details[:dofollow_inbound_count]} without nofollow, #{details[:nofollow_inbound_count]} with nofollow"
112
+ when :only_nofollow_internal_inlinks
113
+ return "all #{details[:nofollow_inbound_count]} incoming internal links use nofollow"
114
+ when :duplicate_title, :duplicate_meta_description
115
+ label = (issue.code == :duplicate_title) ? "same title" : "same meta description"
116
+ return "#{label}: #{details[:value].to_s.inspect}"
117
+ when :duplicate_content_fingerprint
118
+ return "same page text"
119
+ when :duplicate_pages_without_canonical
120
+ return "same page text; at least one page has no canonical link"
121
+ when :near_duplicate_content
122
+ return "text similarity: #{percentage(details[:similarity])}; threshold #{percentage(details[:threshold])}"
123
+ when :near_duplicate_scan_skipped
124
+ return "similar-page check skipped: #{details[:page_count]} pages; limit #{details[:max_pages]}"
125
+ when :structured_data_missing_type
126
+ return "#{issue.message} at #{Array(details[:paths]).join(", ")}" if details[:paths]&.any?
127
+ when :noindex_meta, :nofollow_meta, :noindex_follow_meta, :noindex_nofollow_meta,
128
+ :noindex_header, :nofollow_header, :noindex_follow_header, :noindex_nofollow_header,
129
+ :sitemap_noindex_url
130
+ if details[:content]
131
+ source = details[:name] || details[:source]
132
+ label = source ? "#{source}: " : ""
133
+ return "#{issue.message} (#{label}#{details[:content]})"
134
+ end
135
+ end
136
+ issue.message
137
+ end
138
+
139
+ def related_lines
140
+ details = issue.details || {}
141
+ source_label = (issue.code == :canonical_no_internal_inlinks) ? "canonical on" : "linked from"
142
+ sources = Array(details[:source_urls])
143
+ sources += [details[:source_url]] if details[:source_url]
144
+ relationships = {
145
+ source_label => sources.uniq,
146
+ "links to" => details[:target_urls],
147
+ "image" => details[:image_sources],
148
+ "page" => details[:urls],
149
+ "linked from (without nofollow)" => details[:dofollow_source_urls],
150
+ "linked from (nofollow)" => details[:nofollow_source_urls]
151
+ }
152
+ relationships.flat_map do |label, urls|
153
+ Array(urls).map { |url| "#{label}: #{@urls.call(url)}" }
154
+ end
155
+ end
156
+
157
+ def percentage(value)
158
+ "#{format("%.3f", value * 100).sub(/0+\z/, "").sub(/\.\z/, "")}%"
159
+ end
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "uri"
4
+
5
+ module Crawlscope
6
+ module Reporting
7
+ class UrlFormatter
8
+ def initialize(base_url: nil)
9
+ @base_url = base_url
10
+ end
11
+
12
+ def call(url)
13
+ return url unless url && @base_url
14
+
15
+ uri = URI.parse(url)
16
+ base_uri = URI.parse(@base_url)
17
+
18
+ return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
19
+
20
+ relative = uri.path.to_s.empty? ? "/" : uri.path
21
+ relative += "?#{uri.query}" if uri.query
22
+ relative += "##{uri.fragment}" if uri.fragment
23
+ relative
24
+ rescue URI::InvalidURIError
25
+ url
26
+ end
27
+ end
28
+ end
29
+ end
@@ -9,12 +9,12 @@ module Crawlscope
9
9
  @default_codes = Array(default_codes).map(&:to_sym)
10
10
  end
11
11
 
12
- def self.default(site_name: nil)
12
+ def self.default(site_name: nil, required_types: nil)
13
13
  new(
14
14
  rules: [
15
15
  Rules::Indexability.new,
16
16
  Rules::Metadata.new(site_name: site_name),
17
- Rules::StructuredData.new,
17
+ Rules::StructuredData.new(required_types: required_types),
18
18
  Rules::Uniqueness.new,
19
19
  Rules::ContentQuality.new,
20
20
  Rules::Links.new
@@ -24,6 +24,7 @@ module Crawlscope
24
24
  pages.each do |page|
25
25
  next unless page.html?
26
26
 
27
+ validate_image_alt(page, issues)
27
28
  validate_visible_words(page, issues)
28
29
  validate_visible_text_ratio(page, issues)
29
30
  validate_unique_token_ratio(page, issues)
@@ -32,6 +33,23 @@ module Crawlscope
32
33
 
33
34
  private
34
35
 
36
+ def validate_image_alt(page, issues)
37
+ images = page.doc.css("img:not([alt])")
38
+ return if images.empty?
39
+
40
+ issues.add(
41
+ code: :missing_image_alt,
42
+ severity: :warning,
43
+ category: :content_quality,
44
+ url: page.url,
45
+ message: "images missing alt attributes: #{images.size}",
46
+ details: {
47
+ count: images.size,
48
+ image_sources: images.filter_map { |image| image["src"] }.uniq
49
+ }
50
+ )
51
+ end
52
+
35
53
  def validate_unique_token_ratio(page, issues)
36
54
  tokens = DocumentText.tokens(DocumentText.text_for(page.doc))
37
55
  return if tokens.size < @min_visible_words
@@ -3,32 +3,27 @@
3
3
  module Crawlscope
4
4
  module Rules
5
5
  class Indexability
6
- ROBOTS_META_SELECTOR = 'meta[name="robots"], meta[name="googlebot"]'
7
- X_ROBOTS_TAG_HEADER = "x-robots-tag"
6
+ ROBOTS_META_SELECTOR = Documents::Robots::ROBOTS_META_SELECTOR
7
+ X_ROBOTS_TAG_HEADER = Documents::Robots::X_ROBOTS_TAG_HEADER
8
8
 
9
9
  def self.noindex_header?(headers)
10
- noindex?(header_value(headers, X_ROBOTS_TAG_HEADER))
10
+ Documents::Robots.noindex_header?(headers)
11
11
  end
12
12
 
13
13
  def self.noindex_meta?(doc)
14
- return false unless doc
15
-
16
- doc.css(ROBOTS_META_SELECTOR).any? { |tag| noindex?(tag["content"].to_s) }
14
+ Documents::Robots.noindex_meta?(doc)
17
15
  end
18
16
 
19
17
  def self.header_value(headers, name)
20
- headers.find { |key, _value| key.to_s.casecmp?(name) }&.last.to_s
18
+ Documents::Robots.header_value(headers, name)
21
19
  end
22
20
 
23
21
  def self.directives(value)
24
- value
25
- .split(",")
26
- .map { |directive| directive.split(":", 2).last.to_s.strip }
27
- .reject(&:empty?)
22
+ Documents::Robots.directives(value)
28
23
  end
29
24
 
30
25
  def self.noindex?(value)
31
- directives(value).any? { |directive| directive.casecmp?("noindex") || directive.casecmp?("none") }
26
+ Documents::Robots.new(value).noindex?
32
27
  end
33
28
 
34
29
  attr_reader :code
@@ -65,11 +60,11 @@ module Crawlscope
65
60
  end
66
61
 
67
62
  def follow?(value)
68
- directives(value).any? { |directive| directive.casecmp?("follow") }
63
+ Documents::Robots.new(value).follow?
69
64
  end
70
65
 
71
66
  def nofollow?(value)
72
- directives(value).any? { |directive| directive.casecmp?("nofollow") || directive.casecmp?("none") }
67
+ Documents::Robots.new(value).nofollow?
73
68
  end
74
69
 
75
70
  def validate_meta_robots(page, issues, sitemap_urls)
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class CanonicalChecks
7
+ def initialize(base_url:, targets:, allowed_statuses:, graph:)
8
+ @base_url = base_url
9
+ @base_host = URI.parse(base_url).host
10
+ @targets = targets
11
+ @allowed_statuses = allowed_statuses
12
+ @graph = graph
13
+ end
14
+
15
+ def call(urls:, pages:, issues:)
16
+ sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact
17
+ sitemap_pages = pages.select do |page|
18
+ page.html? && sitemap_urls.include?(page.normalized_url) && page.normalized_url.to_s == page.normalized_final_url.to_s
19
+ end
20
+ return if sitemap_pages.size < 2
21
+
22
+ canonical_entries = canonical_entries_for(sitemap_pages)
23
+ @targets.resolve_all(canonical_entries.map { |entry| entry.fetch(:canonical_url) })
24
+
25
+ canonical_entries.each do |entry|
26
+ page = entry.fetch(:page)
27
+ canonical_url = entry.fetch(:canonical_url)
28
+ canonical_path = entry.fetch(:canonical_path)
29
+ if canonical_path && !root_path?(canonical_path) && @graph.dofollow_count(canonical_url).zero?
30
+ issues.add(
31
+ code: :canonical_no_internal_inlinks,
32
+ severity: :warning,
33
+ category: :links,
34
+ url: canonical_url,
35
+ message: "canonical URL has no incoming internal links without nofollow",
36
+ details: {source_url: page.url}
37
+ )
38
+ end
39
+
40
+ validate_canonical_target_status(page, canonical_url, issues)
41
+ end
42
+ end
43
+
44
+ private
45
+
46
+ def canonical_entries_for(pages)
47
+ pages.filter_map do |page|
48
+ canonical_url = canonical_url_for(page)
49
+ next if canonical_url.nil?
50
+
51
+ target_uri = URI.parse(canonical_url)
52
+ next if target_uri.host != @base_host
53
+
54
+ {
55
+ canonical_path: Url.path(canonical_url),
56
+ canonical_url: canonical_url,
57
+ page: page
58
+ }
59
+ rescue URI::InvalidURIError
60
+ nil
61
+ end
62
+ end
63
+
64
+ def canonical_url_for(page)
65
+ page.canonical.url
66
+ end
67
+
68
+ def root_path?(path)
69
+ path == "/"
70
+ end
71
+
72
+ def validate_canonical_target_status(page, canonical_url, issues)
73
+ target = @targets.fetch(canonical_url)
74
+
75
+ if target.unresolved? || target.ignored_error?
76
+ return
77
+ end
78
+
79
+ if target.redirect?
80
+ issues.add(
81
+ code: :canonical_points_to_redirect,
82
+ severity: :warning,
83
+ category: :metadata,
84
+ url: page.url,
85
+ message: "canonical #{canonical_url} redirects to #{target.final_url}",
86
+ details: {canonical: canonical_url, final_url: target.final_url, status: target.status}
87
+ )
88
+ elsif !target.allowed?(@allowed_statuses)
89
+ issues.add(
90
+ code: :canonical_points_to_error,
91
+ severity: :warning,
92
+ category: :metadata,
93
+ url: page.url,
94
+ message: "canonical #{canonical_url} returns HTTP #{target.status}",
95
+ details: {canonical: canonical_url, status: target.status}
96
+ )
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end