crawlscope 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/UPGRADE.md +76 -0
- data/docs/architecture.md +73 -0
- data/docs/rules.md +49 -0
- data/lib/crawlscope/configuration.rb +2 -1
- data/lib/crawlscope/crawl.rb +11 -79
- data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
- data/lib/crawlscope/crawling/target.rb +45 -0
- data/lib/crawlscope/crawling/target_resolver.rb +75 -0
- data/lib/crawlscope/documents/canonical.rb +22 -0
- data/lib/crawlscope/documents/robots.rb +57 -0
- data/lib/crawlscope/page.rb +4 -0
- data/lib/crawlscope/reporter.rb +6 -160
- data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
- data/lib/crawlscope/reporting/url_formatter.rb +29 -0
- data/lib/crawlscope/rule_registry.rb +2 -2
- data/lib/crawlscope/rules/content_quality.rb +18 -0
- data/lib/crawlscope/rules/indexability.rb +9 -14
- data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
- data/lib/crawlscope/rules/links/extractor.rb +100 -0
- data/lib/crawlscope/rules/links/graph.rb +39 -0
- data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
- data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
- data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
- data/lib/crawlscope/rules/links/scope.rb +23 -0
- data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
- data/lib/crawlscope/rules/links/url_checks.rb +54 -0
- data/lib/crawlscope/rules/links.rb +18 -600
- data/lib/crawlscope/rules/metadata.rb +2 -2
- data/lib/crawlscope/rules/structured_data.rb +26 -77
- data/lib/crawlscope/rules/uniqueness.rb +1 -1
- data/lib/crawlscope/schemas.rb +10 -2
- data/lib/crawlscope/server_timing/reporter.rb +2 -13
- data/lib/crawlscope/structured_data/audit.rb +17 -76
- data/lib/crawlscope/structured_data/check.rb +2 -0
- data/lib/crawlscope/structured_data/document.rb +14 -1
- data/lib/crawlscope/structured_data/entry.rb +31 -0
- data/lib/crawlscope/structured_data/outcome.rb +11 -0
- data/lib/crawlscope/structured_data/report.rb +6 -14
- data/lib/crawlscope/structured_data/reporter.rb +6 -11
- data/lib/crawlscope/structured_data/validator.rb +64 -0
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/cli_test.rb +21 -0
- data/test/crawlscope/content_quality_rule_test.rb +51 -0
- data/test/crawlscope/links_rule_test.rb +16 -0
- data/test/crawlscope/reporter_test.rb +12 -0
- data/test/crawlscope/schema_registry_test.rb +22 -0
- data/test/crawlscope/structured_data_audit_test.rb +60 -0
- data/test/crawlscope/structured_data_report_test.rb +32 -0
- data/test/crawlscope/structured_data_rule_test.rb +11 -1
- data/test/crawlscope/structured_data_validator_test.rb +24 -0
- metadata +22 -1
|
@@ -5,6 +5,8 @@ require "uri"
|
|
|
5
5
|
module Crawlscope
|
|
6
6
|
module Rules
|
|
7
7
|
class Links
|
|
8
|
+
LinkTarget = Crawling::Target
|
|
9
|
+
|
|
8
10
|
LINK_SELECTORS = "a[href]"
|
|
9
11
|
INTERNAL_PATH_PREFIXES_TO_SKIP = ["/rails/", "/cdn-cgi/"].freeze
|
|
10
12
|
LINK_SCHEMES_TO_SKIP = ["mailto:", "tel:", "javascript:", "data:"].freeze
|
|
@@ -18,609 +20,25 @@ module Crawlscope
|
|
|
18
20
|
end
|
|
19
21
|
|
|
20
22
|
def call(urls:, pages:, issues:, context:)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
resolved_links = resolve_links(links, issues)
|
|
34
|
-
validate_nofollow_outgoing_links(resolved_links, issues)
|
|
35
|
-
validate_http_internal_links(links, issues)
|
|
36
|
-
validate_pages_with_no_outgoing_links(urls, pages, links, issues)
|
|
37
|
-
validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
|
|
38
|
-
validate_inbound_counts(urls, pages, resolved_links, issues)
|
|
39
|
-
validate_canonical_targets(urls, pages, resolved_links, issues)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
private
|
|
43
|
-
|
|
44
|
-
def contextual_links(doc)
|
|
45
|
-
doc.css(LINK_SELECTORS)
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
def extract_links(pages)
|
|
49
|
-
html_pages = pages.select(&:html?)
|
|
50
|
-
FetchExecutor.map(
|
|
51
|
-
name: @fetch_executor,
|
|
52
|
-
concurrency: @concurrency,
|
|
53
|
-
items: html_pages
|
|
54
|
-
) { |page| page_links(page) }.flatten
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def page_links(page)
|
|
58
|
-
source_path = Url.path(page.normalized_url)
|
|
59
|
-
return [] unless crawlable_source_path?(source_path)
|
|
60
|
-
|
|
61
|
-
contextual_links(page.doc).filter_map do |node|
|
|
62
|
-
link_for(page: page, source_path: source_path, node: node)
|
|
63
|
-
end
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def link_for(page:, source_path:, node:)
|
|
67
|
-
href = node["href"].to_s.strip
|
|
68
|
-
return unless crawlable_href?(href)
|
|
69
|
-
|
|
70
|
-
anchor_text = normalize_anchor_text(node.text)
|
|
71
|
-
return if anchor_text.empty?
|
|
72
|
-
|
|
73
|
-
target_url = normalize_internal_link(page.normalized_url, href)
|
|
74
|
-
return if target_url.nil?
|
|
75
|
-
|
|
76
|
-
target_path = Url.path(target_url)
|
|
77
|
-
return unless crawlable_path?(target_path)
|
|
78
|
-
|
|
79
|
-
{
|
|
80
|
-
anchor_text: anchor_text,
|
|
81
|
-
http_internal_link: http_internal_link?(page.normalized_url, href),
|
|
82
|
-
nofollow: nofollow_link?(node),
|
|
83
|
-
source_path: source_path,
|
|
84
|
-
source_url: page.normalized_url,
|
|
85
|
-
target_path: target_path,
|
|
86
|
-
target_url: target_url
|
|
87
|
-
}
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
def crawlable_href?(href)
|
|
91
|
-
return false if href.empty?
|
|
92
|
-
return false if href.start_with?("#")
|
|
93
|
-
|
|
94
|
-
LINK_SCHEMES_TO_SKIP.none? { |prefix| href.start_with?(prefix) }
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
def crawlable_path?(path)
|
|
98
|
-
!path.nil? && !skip_internal_path?(path)
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
def normalize_anchor_text(text)
|
|
102
|
-
text.to_s.gsub(/\s+/, " ").strip
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
def nofollow_link?(node)
|
|
106
|
-
node["rel"].to_s.split(/\s+/).any? { |value| value.casecmp?("nofollow") }
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
def http_internal_link?(source_url, href)
|
|
110
|
-
source_uri = URI.parse(source_url.to_s)
|
|
111
|
-
target_uri = URI.parse(URI.join(source_url, href).to_s)
|
|
112
|
-
|
|
113
|
-
source_uri.scheme == "https" && target_uri.scheme == "http" && target_uri.host == @base_host
|
|
114
|
-
rescue URI::InvalidURIError
|
|
115
|
-
false
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
def normalize_internal_link(source_url, href)
|
|
119
|
-
absolute_url = URI.join(source_url, href).to_s
|
|
120
|
-
uri = URI.parse(absolute_url)
|
|
121
|
-
return if uri.host != @base_host
|
|
122
|
-
|
|
123
|
-
uri.fragment = nil
|
|
124
|
-
Url.normalize(uri.to_s, base_url: @base_url)
|
|
125
|
-
rescue URI::InvalidURIError
|
|
126
|
-
nil
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
def report_broken_target(target_url, grouped_links, issues, status)
|
|
130
|
-
source_urls = grouped_links.map { |link| link[:source_url] }.uniq
|
|
131
|
-
issues.add(
|
|
132
|
-
code: :broken_internal_link,
|
|
133
|
-
severity: :warning,
|
|
134
|
-
category: :links,
|
|
135
|
-
url: target_url,
|
|
136
|
-
message: "linked URL returns HTTP #{status}",
|
|
137
|
-
details: {source_urls: source_urls, status: status}
|
|
138
|
-
)
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
def validate_nofollow_outgoing_links(links, issues)
|
|
142
|
-
links.select do |link|
|
|
143
|
-
link[:nofollow] && link[:source_url] != link[:final_url] && !target_for(link[:target_url]).noindex?
|
|
144
|
-
end.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
|
|
145
|
-
target_urls = grouped_links.map { |link| link[:target_url] }.uniq
|
|
146
|
-
|
|
147
|
-
issues.add(
|
|
148
|
-
code: :nofollow_internal_outlinks,
|
|
149
|
-
severity: :warning,
|
|
150
|
-
category: :links,
|
|
151
|
-
url: source_url,
|
|
152
|
-
message: "page has nofollow outgoing internal links",
|
|
153
|
-
details: {target_urls: target_urls}
|
|
154
|
-
)
|
|
155
|
-
end
|
|
156
|
-
end
|
|
157
|
-
|
|
158
|
-
def validate_http_internal_links(links, issues)
|
|
159
|
-
links.select { |link| link[:http_internal_link] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
|
|
160
|
-
target_urls = grouped_links.map { |link| link[:target_url] }.uniq
|
|
161
|
-
|
|
162
|
-
issues.add(
|
|
163
|
-
code: :http_internal_link,
|
|
164
|
-
severity: :warning,
|
|
165
|
-
category: :links,
|
|
166
|
-
url: source_url,
|
|
167
|
-
message: "HTTPS page links to internal HTTP URL",
|
|
168
|
-
details: {target_urls: target_urls}
|
|
169
|
-
)
|
|
170
|
-
end
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
def report_unresolved_target(target_url, grouped_links, issues, resolution)
|
|
174
|
-
source_urls = grouped_links.map { |link| link[:source_url] }.uniq
|
|
175
|
-
suffix = (resolution && resolution[:error]) ? ": #{resolution[:error]}" : ""
|
|
176
|
-
|
|
177
|
-
issues.add(
|
|
178
|
-
code: :unresolved_internal_link,
|
|
179
|
-
severity: :warning,
|
|
180
|
-
category: :links,
|
|
181
|
-
url: target_url,
|
|
182
|
-
message: "cannot check linked URL#{suffix}",
|
|
183
|
-
details: {error: resolution && resolution[:error], source_urls: source_urls}
|
|
184
|
-
)
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
def resolve_links(links, issues)
|
|
188
|
-
resolved_links = []
|
|
189
|
-
grouped_links_by_target = links.group_by { |link| link[:target_url] }
|
|
190
|
-
targets_by_url = resolve_targets(grouped_links_by_target.keys)
|
|
191
|
-
@resolved_targets_by_url.merge!(targets_by_url)
|
|
192
|
-
|
|
193
|
-
grouped_links_by_target.each do |target_url, grouped_links|
|
|
194
|
-
target = target_for(target_url)
|
|
195
|
-
|
|
196
|
-
if target.unresolved?
|
|
197
|
-
report_unresolved_target(target_url, grouped_links, issues, target.resolution)
|
|
198
|
-
next
|
|
199
|
-
end
|
|
200
|
-
|
|
201
|
-
if target.ignored_error?
|
|
202
|
-
next
|
|
203
|
-
end
|
|
204
|
-
|
|
205
|
-
unless target.allowed?(@allowed_statuses)
|
|
206
|
-
report_broken_target(target_url, grouped_links, issues, target.status)
|
|
207
|
-
next
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
report_redirect_target(target_url, grouped_links, issues, target) if target.redirect?
|
|
211
|
-
next unless crawlable_path?(target.final_path)
|
|
212
|
-
|
|
213
|
-
grouped_links.each do |link|
|
|
214
|
-
resolved_links << link.merge(final_path: target.final_path, final_url: target.final_url)
|
|
215
|
-
end
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
resolved_links
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
def report_redirect_target(target_url, grouped_links, issues, target)
|
|
222
|
-
source_urls = grouped_links.map { |link| link[:source_url] }.uniq
|
|
223
|
-
issues.add(
|
|
224
|
-
code: :internal_link_redirects,
|
|
225
|
-
severity: :warning,
|
|
226
|
-
category: :links,
|
|
227
|
-
url: target_url,
|
|
228
|
-
message: "internal link redirects to #{target.final_url}",
|
|
229
|
-
details: {final_url: target.final_url, source_urls: source_urls, status: target.status}
|
|
230
|
-
)
|
|
231
|
-
end
|
|
232
|
-
|
|
233
|
-
def resolve_target(target_url)
|
|
234
|
-
resolution = @resolve_target.call(target_url)
|
|
235
|
-
LinkTarget.new(target_url: target_url, resolution: resolution)
|
|
236
|
-
end
|
|
237
|
-
|
|
238
|
-
def resolve_targets(target_urls)
|
|
239
|
-
if @resolve_targets
|
|
240
|
-
@resolve_targets.call(target_urls).to_h do |target_url, resolution|
|
|
241
|
-
[target_url, LinkTarget.new(target_url: target_url, resolution: resolution)]
|
|
242
|
-
end
|
|
243
|
-
else
|
|
244
|
-
target_urls.to_h { |target_url| [target_url, resolve_target(target_url)] }
|
|
245
|
-
end
|
|
246
|
-
end
|
|
247
|
-
|
|
248
|
-
def target_for(target_url)
|
|
249
|
-
@resolved_targets_by_url.fetch(target_url) do
|
|
250
|
-
target = resolve_target(target_url)
|
|
251
|
-
@resolved_targets_by_url[target_url] = target
|
|
252
|
-
end
|
|
253
|
-
end
|
|
254
|
-
|
|
255
|
-
LinkTarget = Data.define(:target_url, :resolution) do
|
|
256
|
-
def allowed?(statuses)
|
|
257
|
-
statuses.include?(status)
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
def final_path
|
|
261
|
-
Url.path(final_url)
|
|
262
|
-
end
|
|
263
|
-
|
|
264
|
-
def final_url
|
|
265
|
-
value = resolution[:final_url].to_s
|
|
266
|
-
value.empty? ? target_url : value
|
|
267
|
-
end
|
|
268
|
-
|
|
269
|
-
def ignored_error?
|
|
270
|
-
resolution && status.nil? && resolution[:crawled] && resolution[:error]
|
|
271
|
-
end
|
|
272
|
-
|
|
273
|
-
def html?
|
|
274
|
-
resolution && resolution[:html]
|
|
275
|
-
end
|
|
276
|
-
|
|
277
|
-
def noindex?
|
|
278
|
-
Crawlscope::Rules::Indexability.noindex_header?(resolution[:headers] || {}) ||
|
|
279
|
-
Crawlscope::Rules::Indexability.noindex_meta?(resolution[:doc])
|
|
280
|
-
end
|
|
281
|
-
|
|
282
|
-
def status
|
|
283
|
-
resolution && resolution[:status]
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
def redirect?
|
|
287
|
-
(status && (300..399).cover?(status.to_i)) || final_url != target_url
|
|
288
|
-
end
|
|
289
|
-
|
|
290
|
-
def unresolved?
|
|
291
|
-
resolution.nil? || (status.nil? && !ignored_error?)
|
|
292
|
-
end
|
|
293
|
-
end
|
|
294
|
-
|
|
295
|
-
def crawlable_source_path?(path)
|
|
296
|
-
!path.nil? && INTERNAL_PATH_PREFIXES_TO_SKIP.none? { |prefix| path.start_with?(prefix) }
|
|
297
|
-
end
|
|
298
|
-
|
|
299
|
-
def skip_internal_path?(path)
|
|
300
|
-
return true if path == "/"
|
|
301
|
-
|
|
302
|
-
INTERNAL_PATH_PREFIXES_TO_SKIP.any? { |prefix| path.start_with?(prefix) }
|
|
303
|
-
end
|
|
304
|
-
|
|
305
|
-
def validate_inbound_counts(urls, pages, resolved_links, issues)
|
|
306
|
-
sitemap_urls = urls.filter_map do |url|
|
|
307
|
-
normalized_url = Url.normalize(url, base_url: @base_url)
|
|
308
|
-
path = Url.path(normalized_url)
|
|
309
|
-
next if path.nil?
|
|
310
|
-
next if skip_internal_path?(path)
|
|
311
|
-
|
|
312
|
-
normalized_url
|
|
313
|
-
end.to_set
|
|
314
|
-
return if sitemap_urls.size < 2
|
|
315
|
-
|
|
316
|
-
html_urls = pages.each_with_object(Set.new) do |page, result|
|
|
317
|
-
next unless page.html?
|
|
318
|
-
|
|
319
|
-
[page.normalized_url, page.normalized_final_url].compact.each do |url|
|
|
320
|
-
path = Url.path(url)
|
|
321
|
-
next if path.nil?
|
|
322
|
-
next if skip_internal_path?(path)
|
|
323
|
-
|
|
324
|
-
result << url
|
|
325
|
-
end
|
|
326
|
-
end
|
|
327
|
-
|
|
328
|
-
inbound_anchor_counts = Hash.new(0)
|
|
329
|
-
dofollow_inbound_counts = Hash.new(0)
|
|
330
|
-
nofollow_inbound_counts = Hash.new(0)
|
|
331
|
-
sample_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
|
|
332
|
-
dofollow_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
|
|
333
|
-
nofollow_sources_by_target = Hash.new { |hash, key| hash[key] = [] }
|
|
334
|
-
|
|
335
|
-
resolved_links.each do |link|
|
|
336
|
-
target_url = link[:final_url]
|
|
337
|
-
next unless sitemap_urls.include?(target_url)
|
|
338
|
-
next if target_for(link[:target_url]).noindex?
|
|
339
|
-
next if link[:source_url] == target_url
|
|
340
|
-
|
|
341
|
-
inbound_anchor_counts[target_url] += 1
|
|
342
|
-
source_samples = sample_sources_by_target[target_url]
|
|
343
|
-
source_samples << link[:source_url] unless source_samples.include?(link[:source_url])
|
|
344
|
-
|
|
345
|
-
if link[:nofollow]
|
|
346
|
-
nofollow_inbound_counts[target_url] += 1
|
|
347
|
-
nofollow_sources = nofollow_sources_by_target[target_url]
|
|
348
|
-
nofollow_sources << link[:source_url] unless nofollow_sources.include?(link[:source_url])
|
|
349
|
-
else
|
|
350
|
-
dofollow_inbound_counts[target_url] += 1
|
|
351
|
-
dofollow_sources = dofollow_sources_by_target[target_url]
|
|
352
|
-
dofollow_sources << link[:source_url] unless dofollow_sources.include?(link[:source_url])
|
|
353
|
-
end
|
|
354
|
-
end
|
|
355
|
-
|
|
356
|
-
sitemap_urls.each do |target_url|
|
|
357
|
-
next unless html_urls.include?(target_url)
|
|
358
|
-
|
|
359
|
-
inbound_count = inbound_anchor_counts[target_url]
|
|
360
|
-
dofollow_count = dofollow_inbound_counts[target_url]
|
|
361
|
-
nofollow_count = nofollow_inbound_counts[target_url]
|
|
362
|
-
|
|
363
|
-
report_orphan_page(target_url, issues) if inbound_count.zero?
|
|
364
|
-
|
|
365
|
-
if inbound_count.positive? && inbound_count < MIN_INBOUND_ANCHOR_LINKS
|
|
366
|
-
source_samples = sample_sources_by_target[target_url]
|
|
367
|
-
|
|
368
|
-
issues.add(
|
|
369
|
-
code: :low_inbound_anchor_links,
|
|
370
|
-
severity: :warning,
|
|
371
|
-
category: :links,
|
|
372
|
-
url: target_url,
|
|
373
|
-
message: "incoming internal links: #{inbound_count}; minimum #{MIN_INBOUND_ANCHOR_LINKS}",
|
|
374
|
-
details: {inbound_count: inbound_count, minimum: MIN_INBOUND_ANCHOR_LINKS, source_urls: source_samples}
|
|
375
|
-
)
|
|
376
|
-
end
|
|
377
|
-
|
|
378
|
-
report_low_dofollow_inlinks(target_url, dofollow_count, dofollow_sources_by_target, issues)
|
|
379
|
-
report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[target_url], issues)
|
|
380
|
-
report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources_by_target[target_url], dofollow_sources_by_target[target_url], issues)
|
|
381
|
-
end
|
|
382
|
-
end
|
|
383
|
-
|
|
384
|
-
def validate_url_hygiene(urls, links, issues)
|
|
385
|
-
checked_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }
|
|
386
|
-
checked_urls.concat(links.map { |link| link[:target_url] })
|
|
387
|
-
|
|
388
|
-
checked_urls.compact.uniq.each do |url|
|
|
389
|
-
report_url_double_slash(url, issues)
|
|
390
|
-
report_url_too_long(url, issues)
|
|
391
|
-
end
|
|
392
|
-
end
|
|
393
|
-
|
|
394
|
-
def report_url_double_slash(url, issues)
|
|
395
|
-
path = URI.parse(url).path.to_s
|
|
396
|
-
return unless path.match?(%r{//+})
|
|
397
|
-
|
|
398
|
-
issues.add(
|
|
399
|
-
code: :url_double_slash,
|
|
400
|
-
severity: :notice,
|
|
401
|
-
category: :url,
|
|
402
|
-
url: url,
|
|
403
|
-
message: "URL path contains duplicate slashes",
|
|
404
|
-
details: {path: path}
|
|
405
|
-
)
|
|
406
|
-
rescue URI::InvalidURIError
|
|
407
|
-
nil
|
|
408
|
-
end
|
|
409
|
-
|
|
410
|
-
def report_url_too_long(url, issues)
|
|
411
|
-
return unless url.length > 2_048
|
|
412
|
-
|
|
413
|
-
issues.add(
|
|
414
|
-
code: :url_too_long,
|
|
415
|
-
severity: :notice,
|
|
416
|
-
category: :url,
|
|
417
|
-
url: url,
|
|
418
|
-
message: "URL: #{url.length} characters; maximum 2048",
|
|
419
|
-
details: {length: url.length, maximum: 2_048}
|
|
420
|
-
)
|
|
421
|
-
end
|
|
422
|
-
|
|
423
|
-
def validate_pages_with_no_outgoing_links(urls, pages, links, issues)
|
|
424
|
-
sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
|
|
425
|
-
return if sitemap_urls.size < 2
|
|
426
|
-
|
|
427
|
-
source_paths_with_links = links.map { |link| link[:source_path] }.to_set
|
|
428
|
-
|
|
429
|
-
pages.each do |page|
|
|
430
|
-
next unless page.html?
|
|
431
|
-
next unless sitemap_urls.include?(page.normalized_url)
|
|
432
|
-
|
|
433
|
-
source_path = Url.path(page.normalized_url)
|
|
434
|
-
next unless crawlable_source_path?(source_path)
|
|
435
|
-
next if source_paths_with_links.include?(source_path)
|
|
436
|
-
|
|
437
|
-
issues.add(
|
|
438
|
-
code: :page_has_no_outgoing_links,
|
|
439
|
-
severity: :warning,
|
|
440
|
-
category: :links,
|
|
441
|
-
url: page.url,
|
|
442
|
-
message: "page has no outgoing internal links",
|
|
443
|
-
details: {}
|
|
444
|
-
)
|
|
445
|
-
end
|
|
446
|
-
end
|
|
447
|
-
|
|
448
|
-
def validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
|
|
449
|
-
sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
|
|
450
|
-
resolved_links.group_by { |link| link[:final_url] }.each do |final_url, grouped_links|
|
|
451
|
-
link = grouped_links.first
|
|
452
|
-
next if sitemap_urls.include?(final_url)
|
|
453
|
-
next unless crawlable_path?(link[:final_path])
|
|
454
|
-
|
|
455
|
-
target = target_for(final_url)
|
|
456
|
-
next unless target.allowed?(@allowed_statuses) && target.html?
|
|
457
|
-
next if target.noindex?
|
|
458
|
-
|
|
459
|
-
issues.add(
|
|
460
|
-
code: :indexable_page_missing_from_sitemap,
|
|
461
|
-
severity: :warning,
|
|
462
|
-
category: :sitemaps,
|
|
463
|
-
url: final_url,
|
|
464
|
-
message: "linked page is missing from sitemap",
|
|
465
|
-
details: {source_url: link[:source_url], source_urls: grouped_links.map { |entry| entry[:source_url] }.uniq}
|
|
23
|
+
scope = Scope.new
|
|
24
|
+
targets = TargetLookup.new(context: context)
|
|
25
|
+
links = Extractor.new(base_url: context.fetch(:base_url),
|
|
26
|
+
concurrency: context_value(context, :concurrency, default: 1),
|
|
27
|
+
fetch_executor: context_value(context, :fetch_executor), scope: scope).call(pages)
|
|
28
|
+
UrlChecks.new(base_url: context.fetch(:base_url)).call(urls: urls, links: links, issues: issues)
|
|
29
|
+
resolved_links = ResolutionCheck.new(targets: targets,
|
|
30
|
+
allowed_statuses: context.fetch(:allowed_statuses), scope: scope).call(links, issues: issues)
|
|
31
|
+
graph = Graph.new(links: resolved_links, targets: targets)
|
|
32
|
+
OutgoingChecks.new(base_url: context.fetch(:base_url), targets: targets,
|
|
33
|
+
allowed_statuses: context.fetch(:allowed_statuses), scope: scope).call(
|
|
34
|
+
urls: urls, pages: pages, links: links, resolved_links: resolved_links, issues: issues
|
|
466
35
|
)
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
def report_orphan_page(target_url, issues)
|
|
471
|
-
issues.add(
|
|
472
|
-
code: :orphan_page,
|
|
473
|
-
severity: :warning,
|
|
474
|
-
category: :links,
|
|
475
|
-
url: target_url,
|
|
476
|
-
message: "page has no incoming internal links",
|
|
477
|
-
details: {}
|
|
478
|
-
)
|
|
479
|
-
end
|
|
480
|
-
|
|
481
|
-
def report_low_dofollow_inlinks(target_url, dofollow_count, sources_by_target, issues)
|
|
482
|
-
return if dofollow_count.zero?
|
|
483
|
-
return if dofollow_count >= MIN_DOFOLLOW_INBOUND_LINKS
|
|
484
|
-
|
|
485
|
-
source_samples = sources_by_target[target_url]
|
|
486
|
-
|
|
487
|
-
issues.add(
|
|
488
|
-
code: :low_dofollow_inlinks,
|
|
489
|
-
severity: :warning,
|
|
490
|
-
category: :links,
|
|
491
|
-
url: target_url,
|
|
492
|
-
message: "incoming internal links without nofollow: #{dofollow_count}; minimum #{MIN_DOFOLLOW_INBOUND_LINKS}",
|
|
493
|
-
details: {dofollow_inbound_count: dofollow_count, minimum: MIN_DOFOLLOW_INBOUND_LINKS, source_urls: source_samples}
|
|
494
|
-
)
|
|
495
|
-
end
|
|
496
|
-
|
|
497
|
-
def report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, issues)
|
|
498
|
-
return unless nofollow_count.positive? && dofollow_count.zero?
|
|
499
|
-
|
|
500
|
-
issues.add(
|
|
501
|
-
code: :only_nofollow_internal_inlinks,
|
|
502
|
-
severity: :warning,
|
|
503
|
-
category: :links,
|
|
504
|
-
url: target_url,
|
|
505
|
-
message: "all incoming internal links use nofollow",
|
|
506
|
-
details: {nofollow_inbound_count: nofollow_count, source_urls: nofollow_sources}
|
|
507
|
-
)
|
|
508
|
-
end
|
|
509
|
-
|
|
510
|
-
def report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, dofollow_sources, issues)
|
|
511
|
-
return unless nofollow_count.positive? && dofollow_count.positive?
|
|
512
|
-
|
|
513
|
-
issues.add(
|
|
514
|
-
code: :mixed_follow_internal_inlinks,
|
|
515
|
-
severity: :notice,
|
|
516
|
-
category: :links,
|
|
517
|
-
url: target_url,
|
|
518
|
-
message: "page has nofollow and dofollow incoming internal links",
|
|
519
|
-
details: {
|
|
520
|
-
dofollow_inbound_count: dofollow_count,
|
|
521
|
-
nofollow_inbound_count: nofollow_count,
|
|
522
|
-
dofollow_source_urls: dofollow_sources,
|
|
523
|
-
nofollow_source_urls: nofollow_sources
|
|
524
|
-
}
|
|
525
|
-
)
|
|
526
|
-
end
|
|
527
|
-
|
|
528
|
-
def validate_canonical_targets(urls, pages, resolved_links, issues)
|
|
529
|
-
sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact
|
|
530
|
-
sitemap_pages = pages.select do |page|
|
|
531
|
-
page.html? && sitemap_urls.include?(page.normalized_url) && page.normalized_url.to_s == page.normalized_final_url.to_s
|
|
532
|
-
end
|
|
533
|
-
return if sitemap_pages.size < 2
|
|
534
|
-
|
|
535
|
-
dofollow_counts_by_url = dofollow_counts_by_final_url(resolved_links)
|
|
536
|
-
canonical_entries = canonical_entries_for(sitemap_pages)
|
|
537
|
-
@resolved_targets_by_url.merge!(resolve_targets(canonical_entries.map { |entry| entry.fetch(:canonical_url) }))
|
|
538
|
-
|
|
539
|
-
canonical_entries.each do |entry|
|
|
540
|
-
page = entry.fetch(:page)
|
|
541
|
-
canonical_url = entry.fetch(:canonical_url)
|
|
542
|
-
canonical_path = entry.fetch(:canonical_path)
|
|
543
|
-
if canonical_path && !root_path?(canonical_path) && dofollow_counts_by_url[canonical_url].zero?
|
|
544
|
-
issues.add(
|
|
545
|
-
code: :canonical_no_internal_inlinks,
|
|
546
|
-
severity: :warning,
|
|
547
|
-
category: :links,
|
|
548
|
-
url: canonical_url,
|
|
549
|
-
message: "canonical URL has no incoming internal links without nofollow",
|
|
550
|
-
details: {source_url: page.url}
|
|
551
|
-
)
|
|
552
|
-
end
|
|
553
|
-
|
|
554
|
-
validate_canonical_target_status(page, canonical_url, issues)
|
|
555
|
-
end
|
|
36
|
+
InboundChecks.new(base_url: context.fetch(:base_url), graph: graph, scope: scope).call(urls: urls, pages: pages, issues: issues)
|
|
37
|
+
CanonicalChecks.new(base_url: context.fetch(:base_url), targets: targets,
|
|
38
|
+
allowed_statuses: context.fetch(:allowed_statuses), graph: graph).call(urls: urls, pages: pages, issues: issues)
|
|
556
39
|
end
|
|
557
40
|
|
|
558
|
-
|
|
559
|
-
pages.filter_map do |page|
|
|
560
|
-
canonical_url = canonical_url_for(page)
|
|
561
|
-
next if canonical_url.nil?
|
|
562
|
-
|
|
563
|
-
target_uri = URI.parse(canonical_url)
|
|
564
|
-
next if target_uri.host != @base_host
|
|
565
|
-
|
|
566
|
-
{
|
|
567
|
-
canonical_path: Url.path(canonical_url),
|
|
568
|
-
canonical_url: canonical_url,
|
|
569
|
-
page: page
|
|
570
|
-
}
|
|
571
|
-
rescue URI::InvalidURIError
|
|
572
|
-
nil
|
|
573
|
-
end
|
|
574
|
-
end
|
|
575
|
-
|
|
576
|
-
def dofollow_counts_by_final_url(resolved_links)
|
|
577
|
-
resolved_links.each_with_object(Hash.new(0)) do |link, counts|
|
|
578
|
-
next if link[:nofollow]
|
|
579
|
-
next if link[:source_url] == link[:final_url]
|
|
580
|
-
next if target_for(link[:target_url]).noindex?
|
|
581
|
-
|
|
582
|
-
counts[link[:final_url]] += 1
|
|
583
|
-
end
|
|
584
|
-
end
|
|
585
|
-
|
|
586
|
-
def canonical_url_for(page)
|
|
587
|
-
canonical = page.doc.at_css('link[rel="canonical"]')&.[]("href").to_s.strip
|
|
588
|
-
return if canonical.empty?
|
|
589
|
-
|
|
590
|
-
Url.normalize(canonical, base_url: page.url)
|
|
591
|
-
end
|
|
592
|
-
|
|
593
|
-
def root_path?(path)
|
|
594
|
-
path == "/"
|
|
595
|
-
end
|
|
596
|
-
|
|
597
|
-
def validate_canonical_target_status(page, canonical_url, issues)
|
|
598
|
-
target = target_for(canonical_url)
|
|
599
|
-
|
|
600
|
-
if target.unresolved? || target.ignored_error?
|
|
601
|
-
return
|
|
602
|
-
end
|
|
603
|
-
|
|
604
|
-
if target.redirect?
|
|
605
|
-
issues.add(
|
|
606
|
-
code: :canonical_points_to_redirect,
|
|
607
|
-
severity: :warning,
|
|
608
|
-
category: :metadata,
|
|
609
|
-
url: page.url,
|
|
610
|
-
message: "canonical #{canonical_url} redirects to #{target.final_url}",
|
|
611
|
-
details: {canonical: canonical_url, final_url: target.final_url, status: target.status}
|
|
612
|
-
)
|
|
613
|
-
elsif !target.allowed?(@allowed_statuses)
|
|
614
|
-
issues.add(
|
|
615
|
-
code: :canonical_points_to_error,
|
|
616
|
-
severity: :warning,
|
|
617
|
-
category: :metadata,
|
|
618
|
-
url: page.url,
|
|
619
|
-
message: "canonical #{canonical_url} returns HTTP #{target.status}",
|
|
620
|
-
details: {canonical: canonical_url, status: target.status}
|
|
621
|
-
)
|
|
622
|
-
end
|
|
623
|
-
end
|
|
41
|
+
private
|
|
624
42
|
|
|
625
43
|
def context_value(context, name, default: nil)
|
|
626
44
|
if context.respond_to?(name)
|
|
@@ -124,14 +124,14 @@ module Crawlscope
|
|
|
124
124
|
end
|
|
125
125
|
|
|
126
126
|
def validate_canonical(page, issues, sitemap_urls)
|
|
127
|
-
canonical = page.
|
|
127
|
+
canonical = page.canonical.href
|
|
128
128
|
|
|
129
129
|
if canonical.empty?
|
|
130
130
|
issues.add(code: :missing_canonical, severity: :warning, category: :metadata, url: page.url, message: "missing canonical link", details: {})
|
|
131
131
|
return
|
|
132
132
|
end
|
|
133
133
|
|
|
134
|
-
normalized_canonical =
|
|
134
|
+
normalized_canonical = page.canonical.url
|
|
135
135
|
normalized_page_url = Url.normalize(page.url, base_url: page.url)
|
|
136
136
|
return if canonical_matches_page?(normalized_canonical, normalized_page_url)
|
|
137
137
|
|