crawlscope 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/UPGRADE.md +76 -0
- data/docs/architecture.md +73 -0
- data/docs/rules.md +49 -0
- data/lib/crawlscope/configuration.rb +2 -1
- data/lib/crawlscope/crawl.rb +11 -79
- data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
- data/lib/crawlscope/crawling/target.rb +45 -0
- data/lib/crawlscope/crawling/target_resolver.rb +75 -0
- data/lib/crawlscope/documents/canonical.rb +22 -0
- data/lib/crawlscope/documents/robots.rb +57 -0
- data/lib/crawlscope/page.rb +4 -0
- data/lib/crawlscope/reporter.rb +6 -160
- data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
- data/lib/crawlscope/reporting/url_formatter.rb +29 -0
- data/lib/crawlscope/rule_registry.rb +2 -2
- data/lib/crawlscope/rules/content_quality.rb +18 -0
- data/lib/crawlscope/rules/indexability.rb +9 -14
- data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
- data/lib/crawlscope/rules/links/extractor.rb +100 -0
- data/lib/crawlscope/rules/links/graph.rb +39 -0
- data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
- data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
- data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
- data/lib/crawlscope/rules/links/scope.rb +23 -0
- data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
- data/lib/crawlscope/rules/links/url_checks.rb +54 -0
- data/lib/crawlscope/rules/links.rb +18 -600
- data/lib/crawlscope/rules/metadata.rb +2 -2
- data/lib/crawlscope/rules/structured_data.rb +26 -77
- data/lib/crawlscope/rules/uniqueness.rb +1 -1
- data/lib/crawlscope/schemas.rb +10 -2
- data/lib/crawlscope/server_timing/reporter.rb +2 -13
- data/lib/crawlscope/structured_data/audit.rb +17 -76
- data/lib/crawlscope/structured_data/check.rb +2 -0
- data/lib/crawlscope/structured_data/document.rb +14 -1
- data/lib/crawlscope/structured_data/entry.rb +31 -0
- data/lib/crawlscope/structured_data/outcome.rb +11 -0
- data/lib/crawlscope/structured_data/report.rb +6 -14
- data/lib/crawlscope/structured_data/reporter.rb +6 -11
- data/lib/crawlscope/structured_data/validator.rb +64 -0
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/cli_test.rb +21 -0
- data/test/crawlscope/content_quality_rule_test.rb +51 -0
- data/test/crawlscope/links_rule_test.rb +16 -0
- data/test/crawlscope/reporter_test.rb +12 -0
- data/test/crawlscope/schema_registry_test.rb +22 -0
- data/test/crawlscope/structured_data_audit_test.rb +60 -0
- data/test/crawlscope/structured_data_report_test.rb +32 -0
- data/test/crawlscope/structured_data_rule_test.rb +11 -1
- data/test/crawlscope/structured_data_validator_test.rb +24 -0
- metadata +22 -1
data/lib/crawlscope/reporter.rb
CHANGED
|
@@ -1,40 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "uri"
|
|
4
|
-
|
|
5
3
|
module Crawlscope
|
|
6
4
|
class Reporter
|
|
7
|
-
GROUP_LABELS =
|
|
8
|
-
structured_data_schema_error: "Schema errors",
|
|
9
|
-
structured_data_parse_error: "Parse errors",
|
|
10
|
-
structured_data_missing_type: "Missing @type",
|
|
11
|
-
sitemap_redirect_url: "Redirected URLs",
|
|
12
|
-
non_canonical_page_in_sitemap: "URLs with a different canonical",
|
|
13
|
-
indexable_page_missing_from_sitemap: "Linked pages missing from sitemap",
|
|
14
|
-
sitemap_noindex_url: "URLs that block indexing",
|
|
15
|
-
low_dofollow_inlinks: "Too few incoming links without nofollow",
|
|
16
|
-
low_inbound_anchor_links: "Too few incoming links",
|
|
17
|
-
low_unique_token_ratio: "Repeated words",
|
|
18
|
-
low_visible_text_ratio: "Too little text for the HTML size",
|
|
19
|
-
thin_visible_text: "Too few words",
|
|
20
|
-
orphan_page: "Pages with no incoming links",
|
|
21
|
-
nofollow_internal_outlinks: "Outgoing links with nofollow",
|
|
22
|
-
only_nofollow_internal_inlinks: "Incoming links all use nofollow",
|
|
23
|
-
mixed_follow_internal_inlinks: "Incoming links with and without nofollow",
|
|
24
|
-
http_internal_link: "Links from HTTPS to HTTP",
|
|
25
|
-
canonical_no_internal_inlinks: "Canonical URLs with no incoming links without nofollow",
|
|
26
|
-
canonical_points_to_error: "Canonical URLs that return errors",
|
|
27
|
-
url_double_slash: "Repeated slashes in URL paths",
|
|
28
|
-
multiple_h1: "Multiple <h1> tags",
|
|
29
|
-
multiple_title_tags: "Multiple <title> tags",
|
|
30
|
-
incomplete_open_graph_tags: "Missing Open Graph tags",
|
|
31
|
-
title_too_long: "Title exceeds #{Rules::Metadata::TITLE_MAX_LENGTH} characters",
|
|
32
|
-
meta_description_too_long: "Meta description exceeds #{Rules::Metadata::DESCRIPTION_MAX_LENGTH} characters",
|
|
33
|
-
meta_description_too_short: "Meta description below #{Rules::Metadata::DESCRIPTION_MIN_LENGTH} characters",
|
|
34
|
-
duplicate_content_fingerprint: "Duplicate page text",
|
|
35
|
-
near_duplicate_content: "Similar page text",
|
|
36
|
-
near_duplicate_scan_skipped: "Similar-page check skipped"
|
|
37
|
-
}.freeze
|
|
5
|
+
GROUP_LABELS = Reporting::IssuePresenter::GROUP_LABELS
|
|
38
6
|
|
|
39
7
|
def initialize(io:)
|
|
40
8
|
@io = io
|
|
@@ -86,21 +54,14 @@ module Crawlscope
|
|
|
86
54
|
grouped
|
|
87
55
|
.sort_by { |(category, code), grouped_issues| [grouped_issues.map { |issue| severity_order(issue.severity) }.min, -grouped_issues.size, category.to_s, code.to_s] }
|
|
88
56
|
.each do |(category, code), grouped_issues|
|
|
89
|
-
label =
|
|
57
|
+
label = Reporting::IssuePresenter.label(code)
|
|
90
58
|
@io.puts("#{category.to_s.tr("_", " ").capitalize}: #{label} (#{severity_summary_for(grouped_issues)})")
|
|
91
59
|
|
|
60
|
+
urls = Reporting::UrlFormatter.new(base_url: base_url)
|
|
92
61
|
grouped_issues.each do |issue|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
@io.puts(" - #{relative_url(issue.url, base_url: base_url)} #{source}")
|
|
97
|
-
details[:errors].each do |error|
|
|
98
|
-
@io.puts(" #{StructuredData::ErrorFormatter.format(error)}")
|
|
99
|
-
end
|
|
100
|
-
else
|
|
101
|
-
@io.puts(" - #{compact_issue(issue, base_url: base_url)}")
|
|
102
|
-
report_related_urls(issue, details, base_url: base_url)
|
|
103
|
-
end
|
|
62
|
+
first, *related = Reporting::IssuePresenter.new(issue, urls: urls).lines
|
|
63
|
+
@io.puts(" - #{first}")
|
|
64
|
+
related.each { |line| @io.puts(" #{line}") }
|
|
104
65
|
end
|
|
105
66
|
|
|
106
67
|
@io.puts("")
|
|
@@ -116,121 +77,6 @@ module Crawlscope
|
|
|
116
77
|
.map { |severity, entries| "#{entries.size} #{pluralize(severity, entries.size)}" }.join(", ")
|
|
117
78
|
end
|
|
118
79
|
|
|
119
|
-
def compact_issue(issue, base_url:)
|
|
120
|
-
parts = []
|
|
121
|
-
parts << relative_url(issue.url, base_url: base_url) if issue.url
|
|
122
|
-
|
|
123
|
-
detail = compact_detail(issue, base_url: base_url)
|
|
124
|
-
parts << detail unless detail.empty?
|
|
125
|
-
|
|
126
|
-
parts.compact.join(" ")
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
def compact_detail(issue, base_url:)
|
|
130
|
-
details = issue.details || {}
|
|
131
|
-
case issue.code
|
|
132
|
-
when :redirected_page, :sitemap_redirect_url, :internal_link_redirects
|
|
133
|
-
if details[:final_url]
|
|
134
|
-
prefix = (issue.code == :sitemap_redirect_url) ? "sitemap URL redirects to" : "redirects to"
|
|
135
|
-
status = details[:status] ? " (final HTTP #{details[:status]})" : ""
|
|
136
|
-
return "#{prefix} #{relative_url(details[:final_url], base_url: base_url)}#{status}"
|
|
137
|
-
end
|
|
138
|
-
when :canonical_mismatch, :non_canonical_page_in_sitemap
|
|
139
|
-
if details[:canonical]
|
|
140
|
-
prefix = (issue.code == :non_canonical_page_in_sitemap) ? "sitemap lists this URL, but its canonical points to" : "canonical points to a different URL:"
|
|
141
|
-
return "#{prefix} #{relative_url(details[:canonical], base_url: base_url)}"
|
|
142
|
-
end
|
|
143
|
-
when :unexpected_status
|
|
144
|
-
return "HTTP #{details[:status]}" if details[:status]
|
|
145
|
-
when :title_too_long, :meta_description_too_long, :meta_description_too_short
|
|
146
|
-
return "#{details[:length]} characters" if details[:length]
|
|
147
|
-
when :broken_internal_link
|
|
148
|
-
return "linked URL returns HTTP #{details[:status]}"
|
|
149
|
-
when :unresolved_internal_link
|
|
150
|
-
return ["cannot check linked URL", details[:error]].compact.join(": ")
|
|
151
|
-
when :indexable_page_missing_from_sitemap
|
|
152
|
-
return "linked page is missing from sitemap"
|
|
153
|
-
when :canonical_points_to_redirect
|
|
154
|
-
status = details[:status] ? " (final HTTP #{details[:status]})" : ""
|
|
155
|
-
return "canonical #{relative_url(details[:canonical], base_url: base_url)} redirects to #{relative_url(details[:final_url], base_url: base_url)}#{status}"
|
|
156
|
-
when :canonical_points_to_error
|
|
157
|
-
return "canonical #{relative_url(details[:canonical], base_url: base_url)} returns HTTP #{details[:status]}"
|
|
158
|
-
when :low_dofollow_inlinks, :low_inbound_anchor_links
|
|
159
|
-
count = details[:dofollow_inbound_count] || details[:inbound_count]
|
|
160
|
-
label = (issue.code == :low_dofollow_inlinks) ? "incoming links without nofollow" : "incoming links"
|
|
161
|
-
return "#{label}: #{count}; minimum #{details[:minimum]}"
|
|
162
|
-
when :low_unique_token_ratio, :low_visible_text_ratio
|
|
163
|
-
label = (issue.code == :low_unique_token_ratio) ? "unique words" : "visible text / HTML bytes"
|
|
164
|
-
return "#{label}: #{percentage(details[:ratio])}; minimum #{percentage(details[:threshold])}"
|
|
165
|
-
when :thin_visible_text
|
|
166
|
-
return "visible words: #{details[:word_count]}; minimum #{details[:minimum]}"
|
|
167
|
-
when :mixed_follow_internal_inlinks
|
|
168
|
-
return "incoming links: #{details[:dofollow_inbound_count]} without nofollow, #{details[:nofollow_inbound_count]} with nofollow"
|
|
169
|
-
when :only_nofollow_internal_inlinks
|
|
170
|
-
return "all #{details[:nofollow_inbound_count]} incoming internal links use nofollow"
|
|
171
|
-
when :duplicate_title, :duplicate_meta_description
|
|
172
|
-
label = (issue.code == :duplicate_title) ? "same title" : "same meta description"
|
|
173
|
-
return "#{label}: #{details[:value].to_s.inspect}"
|
|
174
|
-
when :duplicate_content_fingerprint
|
|
175
|
-
return "same page text"
|
|
176
|
-
when :duplicate_pages_without_canonical
|
|
177
|
-
return "same page text; at least one page has no canonical link"
|
|
178
|
-
when :near_duplicate_content
|
|
179
|
-
return "text similarity: #{percentage(details[:similarity])}; threshold #{percentage(details[:threshold])}"
|
|
180
|
-
when :near_duplicate_scan_skipped
|
|
181
|
-
return "similar-page check skipped: #{details[:page_count]} pages; limit #{details[:max_pages]}"
|
|
182
|
-
when :structured_data_missing_type
|
|
183
|
-
return "#{issue.message} at #{Array(details[:paths]).join(", ")}" if details[:paths]&.any?
|
|
184
|
-
when :noindex_meta, :nofollow_meta, :noindex_follow_meta, :noindex_nofollow_meta,
|
|
185
|
-
:noindex_header, :nofollow_header, :noindex_follow_header, :noindex_nofollow_header,
|
|
186
|
-
:sitemap_noindex_url
|
|
187
|
-
if details[:content]
|
|
188
|
-
source = details[:name] || details[:source]
|
|
189
|
-
label = source ? "#{source}: " : ""
|
|
190
|
-
return "#{issue.message} (#{label}#{details[:content]})"
|
|
191
|
-
end
|
|
192
|
-
end
|
|
193
|
-
issue.message
|
|
194
|
-
end
|
|
195
|
-
|
|
196
|
-
def report_related_urls(issue, details, base_url:)
|
|
197
|
-
source_label = (issue.code == :canonical_no_internal_inlinks) ? "canonical on" : "linked from"
|
|
198
|
-
sources = Array(details[:source_urls])
|
|
199
|
-
sources += [details[:source_url]] if details[:source_url]
|
|
200
|
-
relationships = {
|
|
201
|
-
source_label => sources.uniq,
|
|
202
|
-
"links to" => details[:target_urls],
|
|
203
|
-
"page" => details[:urls],
|
|
204
|
-
"linked from (without nofollow)" => details[:dofollow_source_urls],
|
|
205
|
-
"linked from (nofollow)" => details[:nofollow_source_urls]
|
|
206
|
-
}
|
|
207
|
-
relationships.each do |label, urls|
|
|
208
|
-
Array(urls).each do |url|
|
|
209
|
-
@io.puts(" #{label}: #{relative_url(url, base_url: base_url)}")
|
|
210
|
-
end
|
|
211
|
-
end
|
|
212
|
-
end
|
|
213
|
-
|
|
214
|
-
def relative_url(url, base_url:)
|
|
215
|
-
return url unless url && base_url
|
|
216
|
-
|
|
217
|
-
uri = URI.parse(url)
|
|
218
|
-
base_uri = URI.parse(base_url)
|
|
219
|
-
|
|
220
|
-
return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
|
|
221
|
-
|
|
222
|
-
relative = uri.path.to_s.empty? ? "/" : uri.path
|
|
223
|
-
relative += "?#{uri.query}" if uri.query
|
|
224
|
-
relative += "##{uri.fragment}" if uri.fragment
|
|
225
|
-
relative
|
|
226
|
-
rescue URI::InvalidURIError
|
|
227
|
-
url
|
|
228
|
-
end
|
|
229
|
-
|
|
230
|
-
def percentage(value)
|
|
231
|
-
"#{format("%.3f", value * 100).sub(/0+\z/, "").sub(/\.\z/, "")}%"
|
|
232
|
-
end
|
|
233
|
-
|
|
234
80
|
def pluralize(word, count)
|
|
235
81
|
return word.to_s if count == 1
|
|
236
82
|
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Reporting
|
|
5
|
+
class IssuePresenter
|
|
6
|
+
GROUP_LABELS = {
|
|
7
|
+
structured_data_schema_error: "Schema errors",
|
|
8
|
+
structured_data_parse_error: "Parse errors",
|
|
9
|
+
structured_data_missing_type: "Missing @type",
|
|
10
|
+
sitemap_redirect_url: "Redirected URLs",
|
|
11
|
+
non_canonical_page_in_sitemap: "URLs with a different canonical",
|
|
12
|
+
indexable_page_missing_from_sitemap: "Linked pages missing from sitemap",
|
|
13
|
+
sitemap_noindex_url: "URLs that block indexing",
|
|
14
|
+
low_dofollow_inlinks: "Too few incoming links without nofollow",
|
|
15
|
+
low_inbound_anchor_links: "Too few incoming links",
|
|
16
|
+
low_unique_token_ratio: "Repeated words",
|
|
17
|
+
low_visible_text_ratio: "Too little text for the HTML size",
|
|
18
|
+
thin_visible_text: "Too few words",
|
|
19
|
+
missing_image_alt: "Images missing alt attributes",
|
|
20
|
+
orphan_page: "Pages with no incoming links",
|
|
21
|
+
nofollow_internal_outlinks: "Outgoing links with nofollow",
|
|
22
|
+
only_nofollow_internal_inlinks: "Incoming links all use nofollow",
|
|
23
|
+
mixed_follow_internal_inlinks: "Incoming links with and without nofollow",
|
|
24
|
+
http_internal_link: "Links from HTTPS to HTTP",
|
|
25
|
+
canonical_no_internal_inlinks: "Canonical URLs with no incoming links without nofollow",
|
|
26
|
+
canonical_points_to_error: "Canonical URLs that return errors",
|
|
27
|
+
url_double_slash: "Repeated slashes in URL paths",
|
|
28
|
+
multiple_h1: "Multiple <h1> tags",
|
|
29
|
+
multiple_title_tags: "Multiple <title> tags",
|
|
30
|
+
incomplete_open_graph_tags: "Missing Open Graph tags",
|
|
31
|
+
title_too_long: "Title exceeds #{Rules::Metadata::TITLE_MAX_LENGTH} characters",
|
|
32
|
+
meta_description_too_long: "Meta description exceeds #{Rules::Metadata::DESCRIPTION_MAX_LENGTH} characters",
|
|
33
|
+
meta_description_too_short: "Meta description below #{Rules::Metadata::DESCRIPTION_MIN_LENGTH} characters",
|
|
34
|
+
duplicate_content_fingerprint: "Duplicate page text",
|
|
35
|
+
near_duplicate_content: "Similar page text",
|
|
36
|
+
near_duplicate_scan_skipped: "Similar-page check skipped"
|
|
37
|
+
}.freeze
|
|
38
|
+
|
|
39
|
+
def initialize(issue, urls:)
|
|
40
|
+
@issue = issue
|
|
41
|
+
@urls = urls
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def self.label(code)
|
|
45
|
+
GROUP_LABELS.fetch(code) { code.to_s.tr("_", " ").capitalize }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def lines
|
|
49
|
+
details = issue.details || {}
|
|
50
|
+
if issue.code == :structured_data_schema_error && details[:errors]&.any?
|
|
51
|
+
source = (details[:source] == "json-ld") ? "JSON-LD" : details[:source]
|
|
52
|
+
["#{@urls.call(issue.url)} #{source}", *details[:errors].map { |error| StructuredData::ErrorFormatter.format(error) }]
|
|
53
|
+
else
|
|
54
|
+
[summary, *related_lines]
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
attr_reader :issue
|
|
61
|
+
|
|
62
|
+
def summary
|
|
63
|
+
parts = []
|
|
64
|
+
parts << @urls.call(issue.url) if issue.url
|
|
65
|
+
|
|
66
|
+
text = detail
|
|
67
|
+
parts << text unless text.empty?
|
|
68
|
+
|
|
69
|
+
parts.compact.join(" ")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def detail
|
|
73
|
+
details = issue.details || {}
|
|
74
|
+
case issue.code
|
|
75
|
+
when :redirected_page, :sitemap_redirect_url, :internal_link_redirects
|
|
76
|
+
if details[:final_url]
|
|
77
|
+
prefix = (issue.code == :sitemap_redirect_url) ? "sitemap URL redirects to" : "redirects to"
|
|
78
|
+
status = details[:status] ? " (final HTTP #{details[:status]})" : ""
|
|
79
|
+
return "#{prefix} #{@urls.call(details[:final_url])}#{status}"
|
|
80
|
+
end
|
|
81
|
+
when :canonical_mismatch, :non_canonical_page_in_sitemap
|
|
82
|
+
if details[:canonical]
|
|
83
|
+
prefix = (issue.code == :non_canonical_page_in_sitemap) ? "sitemap lists this URL, but its canonical points to" : "canonical points to a different URL:"
|
|
84
|
+
return "#{prefix} #{@urls.call(details[:canonical])}"
|
|
85
|
+
end
|
|
86
|
+
when :unexpected_status
|
|
87
|
+
return "HTTP #{details[:status]}" if details[:status]
|
|
88
|
+
when :title_too_long, :meta_description_too_long, :meta_description_too_short
|
|
89
|
+
return "#{details[:length]} characters" if details[:length]
|
|
90
|
+
when :broken_internal_link
|
|
91
|
+
return "linked URL returns HTTP #{details[:status]}"
|
|
92
|
+
when :unresolved_internal_link
|
|
93
|
+
return ["cannot check linked URL", details[:error]].compact.join(": ")
|
|
94
|
+
when :indexable_page_missing_from_sitemap
|
|
95
|
+
return "linked page is missing from sitemap"
|
|
96
|
+
when :canonical_points_to_redirect
|
|
97
|
+
status = details[:status] ? " (final HTTP #{details[:status]})" : ""
|
|
98
|
+
return "canonical #{@urls.call(details[:canonical])} redirects to #{@urls.call(details[:final_url])}#{status}"
|
|
99
|
+
when :canonical_points_to_error
|
|
100
|
+
return "canonical #{@urls.call(details[:canonical])} returns HTTP #{details[:status]}"
|
|
101
|
+
when :low_dofollow_inlinks, :low_inbound_anchor_links
|
|
102
|
+
count = details[:dofollow_inbound_count] || details[:inbound_count]
|
|
103
|
+
label = (issue.code == :low_dofollow_inlinks) ? "incoming links without nofollow" : "incoming links"
|
|
104
|
+
return "#{label}: #{count}; minimum #{details[:minimum]}"
|
|
105
|
+
when :low_unique_token_ratio, :low_visible_text_ratio
|
|
106
|
+
label = (issue.code == :low_unique_token_ratio) ? "unique words" : "visible text / HTML bytes"
|
|
107
|
+
return "#{label}: #{percentage(details[:ratio])}; minimum #{percentage(details[:threshold])}"
|
|
108
|
+
when :thin_visible_text
|
|
109
|
+
return "visible words: #{details[:word_count]}; minimum #{details[:minimum]}"
|
|
110
|
+
when :mixed_follow_internal_inlinks
|
|
111
|
+
return "incoming links: #{details[:dofollow_inbound_count]} without nofollow, #{details[:nofollow_inbound_count]} with nofollow"
|
|
112
|
+
when :only_nofollow_internal_inlinks
|
|
113
|
+
return "all #{details[:nofollow_inbound_count]} incoming internal links use nofollow"
|
|
114
|
+
when :duplicate_title, :duplicate_meta_description
|
|
115
|
+
label = (issue.code == :duplicate_title) ? "same title" : "same meta description"
|
|
116
|
+
return "#{label}: #{details[:value].to_s.inspect}"
|
|
117
|
+
when :duplicate_content_fingerprint
|
|
118
|
+
return "same page text"
|
|
119
|
+
when :duplicate_pages_without_canonical
|
|
120
|
+
return "same page text; at least one page has no canonical link"
|
|
121
|
+
when :near_duplicate_content
|
|
122
|
+
return "text similarity: #{percentage(details[:similarity])}; threshold #{percentage(details[:threshold])}"
|
|
123
|
+
when :near_duplicate_scan_skipped
|
|
124
|
+
return "similar-page check skipped: #{details[:page_count]} pages; limit #{details[:max_pages]}"
|
|
125
|
+
when :structured_data_missing_type
|
|
126
|
+
return "#{issue.message} at #{Array(details[:paths]).join(", ")}" if details[:paths]&.any?
|
|
127
|
+
when :noindex_meta, :nofollow_meta, :noindex_follow_meta, :noindex_nofollow_meta,
|
|
128
|
+
:noindex_header, :nofollow_header, :noindex_follow_header, :noindex_nofollow_header,
|
|
129
|
+
:sitemap_noindex_url
|
|
130
|
+
if details[:content]
|
|
131
|
+
source = details[:name] || details[:source]
|
|
132
|
+
label = source ? "#{source}: " : ""
|
|
133
|
+
return "#{issue.message} (#{label}#{details[:content]})"
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
issue.message
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def related_lines
|
|
140
|
+
details = issue.details || {}
|
|
141
|
+
source_label = (issue.code == :canonical_no_internal_inlinks) ? "canonical on" : "linked from"
|
|
142
|
+
sources = Array(details[:source_urls])
|
|
143
|
+
sources += [details[:source_url]] if details[:source_url]
|
|
144
|
+
relationships = {
|
|
145
|
+
source_label => sources.uniq,
|
|
146
|
+
"links to" => details[:target_urls],
|
|
147
|
+
"image" => details[:image_sources],
|
|
148
|
+
"page" => details[:urls],
|
|
149
|
+
"linked from (without nofollow)" => details[:dofollow_source_urls],
|
|
150
|
+
"linked from (nofollow)" => details[:nofollow_source_urls]
|
|
151
|
+
}
|
|
152
|
+
relationships.flat_map do |label, urls|
|
|
153
|
+
Array(urls).map { |url| "#{label}: #{@urls.call(url)}" }
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def percentage(value)
|
|
158
|
+
"#{format("%.3f", value * 100).sub(/0+\z/, "").sub(/\.\z/, "")}%"
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
|
|
5
|
+
module Crawlscope
|
|
6
|
+
module Reporting
|
|
7
|
+
class UrlFormatter
|
|
8
|
+
def initialize(base_url: nil)
|
|
9
|
+
@base_url = base_url
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def call(url)
|
|
13
|
+
return url unless url && @base_url
|
|
14
|
+
|
|
15
|
+
uri = URI.parse(url)
|
|
16
|
+
base_uri = URI.parse(@base_url)
|
|
17
|
+
|
|
18
|
+
return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
|
|
19
|
+
|
|
20
|
+
relative = uri.path.to_s.empty? ? "/" : uri.path
|
|
21
|
+
relative += "?#{uri.query}" if uri.query
|
|
22
|
+
relative += "##{uri.fragment}" if uri.fragment
|
|
23
|
+
relative
|
|
24
|
+
rescue URI::InvalidURIError
|
|
25
|
+
url
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -9,12 +9,12 @@ module Crawlscope
|
|
|
9
9
|
@default_codes = Array(default_codes).map(&:to_sym)
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
-
def self.default(site_name: nil)
|
|
12
|
+
def self.default(site_name: nil, required_types: nil)
|
|
13
13
|
new(
|
|
14
14
|
rules: [
|
|
15
15
|
Rules::Indexability.new,
|
|
16
16
|
Rules::Metadata.new(site_name: site_name),
|
|
17
|
-
Rules::StructuredData.new,
|
|
17
|
+
Rules::StructuredData.new(required_types: required_types),
|
|
18
18
|
Rules::Uniqueness.new,
|
|
19
19
|
Rules::ContentQuality.new,
|
|
20
20
|
Rules::Links.new
|
|
@@ -24,6 +24,7 @@ module Crawlscope
|
|
|
24
24
|
pages.each do |page|
|
|
25
25
|
next unless page.html?
|
|
26
26
|
|
|
27
|
+
validate_image_alt(page, issues)
|
|
27
28
|
validate_visible_words(page, issues)
|
|
28
29
|
validate_visible_text_ratio(page, issues)
|
|
29
30
|
validate_unique_token_ratio(page, issues)
|
|
@@ -32,6 +33,23 @@ module Crawlscope
|
|
|
32
33
|
|
|
33
34
|
private
|
|
34
35
|
|
|
36
|
+
def validate_image_alt(page, issues)
|
|
37
|
+
images = page.doc.css("img:not([alt])")
|
|
38
|
+
return if images.empty?
|
|
39
|
+
|
|
40
|
+
issues.add(
|
|
41
|
+
code: :missing_image_alt,
|
|
42
|
+
severity: :warning,
|
|
43
|
+
category: :content_quality,
|
|
44
|
+
url: page.url,
|
|
45
|
+
message: "images missing alt attributes: #{images.size}",
|
|
46
|
+
details: {
|
|
47
|
+
count: images.size,
|
|
48
|
+
image_sources: images.filter_map { |image| image["src"] }.uniq
|
|
49
|
+
}
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
|
|
35
53
|
def validate_unique_token_ratio(page, issues)
|
|
36
54
|
tokens = DocumentText.tokens(DocumentText.text_for(page.doc))
|
|
37
55
|
return if tokens.size < @min_visible_words
|
|
@@ -3,32 +3,27 @@
|
|
|
3
3
|
module Crawlscope
|
|
4
4
|
module Rules
|
|
5
5
|
class Indexability
|
|
6
|
-
ROBOTS_META_SELECTOR =
|
|
7
|
-
X_ROBOTS_TAG_HEADER =
|
|
6
|
+
ROBOTS_META_SELECTOR = Documents::Robots::ROBOTS_META_SELECTOR
|
|
7
|
+
X_ROBOTS_TAG_HEADER = Documents::Robots::X_ROBOTS_TAG_HEADER
|
|
8
8
|
|
|
9
9
|
def self.noindex_header?(headers)
|
|
10
|
-
|
|
10
|
+
Documents::Robots.noindex_header?(headers)
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def self.noindex_meta?(doc)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
doc.css(ROBOTS_META_SELECTOR).any? { |tag| noindex?(tag["content"].to_s) }
|
|
14
|
+
Documents::Robots.noindex_meta?(doc)
|
|
17
15
|
end
|
|
18
16
|
|
|
19
17
|
def self.header_value(headers, name)
|
|
20
|
-
headers
|
|
18
|
+
Documents::Robots.header_value(headers, name)
|
|
21
19
|
end
|
|
22
20
|
|
|
23
21
|
def self.directives(value)
|
|
24
|
-
value
|
|
25
|
-
.split(",")
|
|
26
|
-
.map { |directive| directive.split(":", 2).last.to_s.strip }
|
|
27
|
-
.reject(&:empty?)
|
|
22
|
+
Documents::Robots.directives(value)
|
|
28
23
|
end
|
|
29
24
|
|
|
30
25
|
def self.noindex?(value)
|
|
31
|
-
|
|
26
|
+
Documents::Robots.new(value).noindex?
|
|
32
27
|
end
|
|
33
28
|
|
|
34
29
|
attr_reader :code
|
|
@@ -65,11 +60,11 @@ module Crawlscope
|
|
|
65
60
|
end
|
|
66
61
|
|
|
67
62
|
def follow?(value)
|
|
68
|
-
|
|
63
|
+
Documents::Robots.new(value).follow?
|
|
69
64
|
end
|
|
70
65
|
|
|
71
66
|
def nofollow?(value)
|
|
72
|
-
|
|
67
|
+
Documents::Robots.new(value).nofollow?
|
|
73
68
|
end
|
|
74
69
|
|
|
75
70
|
def validate_meta_robots(page, issues, sitemap_urls)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class CanonicalChecks
|
|
7
|
+
def initialize(base_url:, targets:, allowed_statuses:, graph:)
|
|
8
|
+
@base_url = base_url
|
|
9
|
+
@base_host = URI.parse(base_url).host
|
|
10
|
+
@targets = targets
|
|
11
|
+
@allowed_statuses = allowed_statuses
|
|
12
|
+
@graph = graph
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(urls:, pages:, issues:)
|
|
16
|
+
sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact
|
|
17
|
+
sitemap_pages = pages.select do |page|
|
|
18
|
+
page.html? && sitemap_urls.include?(page.normalized_url) && page.normalized_url.to_s == page.normalized_final_url.to_s
|
|
19
|
+
end
|
|
20
|
+
return if sitemap_pages.size < 2
|
|
21
|
+
|
|
22
|
+
canonical_entries = canonical_entries_for(sitemap_pages)
|
|
23
|
+
@targets.resolve_all(canonical_entries.map { |entry| entry.fetch(:canonical_url) })
|
|
24
|
+
|
|
25
|
+
canonical_entries.each do |entry|
|
|
26
|
+
page = entry.fetch(:page)
|
|
27
|
+
canonical_url = entry.fetch(:canonical_url)
|
|
28
|
+
canonical_path = entry.fetch(:canonical_path)
|
|
29
|
+
if canonical_path && !root_path?(canonical_path) && @graph.dofollow_count(canonical_url).zero?
|
|
30
|
+
issues.add(
|
|
31
|
+
code: :canonical_no_internal_inlinks,
|
|
32
|
+
severity: :warning,
|
|
33
|
+
category: :links,
|
|
34
|
+
url: canonical_url,
|
|
35
|
+
message: "canonical URL has no incoming internal links without nofollow",
|
|
36
|
+
details: {source_url: page.url}
|
|
37
|
+
)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
validate_canonical_target_status(page, canonical_url, issues)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def canonical_entries_for(pages)
|
|
47
|
+
pages.filter_map do |page|
|
|
48
|
+
canonical_url = canonical_url_for(page)
|
|
49
|
+
next if canonical_url.nil?
|
|
50
|
+
|
|
51
|
+
target_uri = URI.parse(canonical_url)
|
|
52
|
+
next if target_uri.host != @base_host
|
|
53
|
+
|
|
54
|
+
{
|
|
55
|
+
canonical_path: Url.path(canonical_url),
|
|
56
|
+
canonical_url: canonical_url,
|
|
57
|
+
page: page
|
|
58
|
+
}
|
|
59
|
+
rescue URI::InvalidURIError
|
|
60
|
+
nil
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def canonical_url_for(page)
|
|
65
|
+
page.canonical.url
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def root_path?(path)
|
|
69
|
+
path == "/"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def validate_canonical_target_status(page, canonical_url, issues)
|
|
73
|
+
target = @targets.fetch(canonical_url)
|
|
74
|
+
|
|
75
|
+
if target.unresolved? || target.ignored_error?
|
|
76
|
+
return
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
if target.redirect?
|
|
80
|
+
issues.add(
|
|
81
|
+
code: :canonical_points_to_redirect,
|
|
82
|
+
severity: :warning,
|
|
83
|
+
category: :metadata,
|
|
84
|
+
url: page.url,
|
|
85
|
+
message: "canonical #{canonical_url} redirects to #{target.final_url}",
|
|
86
|
+
details: {canonical: canonical_url, final_url: target.final_url, status: target.status}
|
|
87
|
+
)
|
|
88
|
+
elsif !target.allowed?(@allowed_statuses)
|
|
89
|
+
issues.add(
|
|
90
|
+
code: :canonical_points_to_error,
|
|
91
|
+
severity: :warning,
|
|
92
|
+
category: :metadata,
|
|
93
|
+
url: page.url,
|
|
94
|
+
message: "canonical #{canonical_url} returns HTTP #{target.status}",
|
|
95
|
+
details: {canonical: canonical_url, status: target.status}
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|