crawlscope 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/UPGRADE.md +76 -0
- data/docs/architecture.md +73 -0
- data/docs/rules.md +49 -0
- data/lib/crawlscope/configuration.rb +2 -1
- data/lib/crawlscope/crawl.rb +11 -79
- data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
- data/lib/crawlscope/crawling/target.rb +45 -0
- data/lib/crawlscope/crawling/target_resolver.rb +75 -0
- data/lib/crawlscope/documents/canonical.rb +22 -0
- data/lib/crawlscope/documents/robots.rb +57 -0
- data/lib/crawlscope/page.rb +4 -0
- data/lib/crawlscope/reporter.rb +6 -160
- data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
- data/lib/crawlscope/reporting/url_formatter.rb +29 -0
- data/lib/crawlscope/rule_registry.rb +2 -2
- data/lib/crawlscope/rules/content_quality.rb +18 -0
- data/lib/crawlscope/rules/indexability.rb +9 -14
- data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
- data/lib/crawlscope/rules/links/extractor.rb +100 -0
- data/lib/crawlscope/rules/links/graph.rb +39 -0
- data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
- data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
- data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
- data/lib/crawlscope/rules/links/scope.rb +23 -0
- data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
- data/lib/crawlscope/rules/links/url_checks.rb +54 -0
- data/lib/crawlscope/rules/links.rb +18 -600
- data/lib/crawlscope/rules/metadata.rb +2 -2
- data/lib/crawlscope/rules/structured_data.rb +26 -77
- data/lib/crawlscope/rules/uniqueness.rb +1 -1
- data/lib/crawlscope/schemas.rb +10 -2
- data/lib/crawlscope/server_timing/reporter.rb +2 -13
- data/lib/crawlscope/structured_data/audit.rb +17 -76
- data/lib/crawlscope/structured_data/check.rb +2 -0
- data/lib/crawlscope/structured_data/document.rb +14 -1
- data/lib/crawlscope/structured_data/entry.rb +31 -0
- data/lib/crawlscope/structured_data/outcome.rb +11 -0
- data/lib/crawlscope/structured_data/report.rb +6 -14
- data/lib/crawlscope/structured_data/reporter.rb +6 -11
- data/lib/crawlscope/structured_data/validator.rb +64 -0
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/cli_test.rb +21 -0
- data/test/crawlscope/content_quality_rule_test.rb +51 -0
- data/test/crawlscope/links_rule_test.rb +16 -0
- data/test/crawlscope/reporter_test.rb +12 -0
- data/test/crawlscope/schema_registry_test.rb +22 -0
- data/test/crawlscope/structured_data_audit_test.rb +60 -0
- data/test/crawlscope/structured_data_report_test.rb +32 -0
- data/test/crawlscope/structured_data_rule_test.rb +11 -1
- data/test/crawlscope/structured_data_validator_test.rb +24 -0
- metadata +22 -1
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class Extractor
|
|
7
|
+
def initialize(base_url:, concurrency:, fetch_executor:, scope:)
|
|
8
|
+
@base_url = base_url
|
|
9
|
+
@base_host = URI.parse(base_url).host
|
|
10
|
+
@concurrency = concurrency
|
|
11
|
+
@fetch_executor = fetch_executor
|
|
12
|
+
@scope = scope
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(pages)
|
|
16
|
+
html_pages = pages.select(&:html?)
|
|
17
|
+
FetchExecutor.map(
|
|
18
|
+
name: @fetch_executor,
|
|
19
|
+
concurrency: @concurrency,
|
|
20
|
+
items: html_pages
|
|
21
|
+
) { |page| page_links(page) }.flatten
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def page_links(page)
|
|
27
|
+
source_path = Url.path(page.normalized_url)
|
|
28
|
+
return [] unless @scope.crawlable_source_path?(source_path)
|
|
29
|
+
|
|
30
|
+
contextual_links(page.doc).filter_map do |node|
|
|
31
|
+
link_for(page: page, source_path: source_path, node: node)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def contextual_links(doc)
|
|
36
|
+
doc.css(LINK_SELECTORS)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def link_for(page:, source_path:, node:)
|
|
40
|
+
href = node["href"].to_s.strip
|
|
41
|
+
return unless crawlable_href?(href)
|
|
42
|
+
|
|
43
|
+
anchor_text = normalize_anchor_text(node.text)
|
|
44
|
+
return if anchor_text.empty?
|
|
45
|
+
|
|
46
|
+
target_url = normalize_internal_link(page.normalized_url, href)
|
|
47
|
+
return if target_url.nil?
|
|
48
|
+
|
|
49
|
+
target_path = Url.path(target_url)
|
|
50
|
+
return unless @scope.crawlable_path?(target_path)
|
|
51
|
+
|
|
52
|
+
{
|
|
53
|
+
anchor_text: anchor_text,
|
|
54
|
+
http_internal_link: http_internal_link?(page.normalized_url, href),
|
|
55
|
+
nofollow: nofollow_link?(node),
|
|
56
|
+
source_path: source_path,
|
|
57
|
+
source_url: page.normalized_url,
|
|
58
|
+
target_path: target_path,
|
|
59
|
+
target_url: target_url
|
|
60
|
+
}
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def crawlable_href?(href)
|
|
64
|
+
return false if href.empty?
|
|
65
|
+
return false if href.start_with?("#")
|
|
66
|
+
|
|
67
|
+
LINK_SCHEMES_TO_SKIP.none? { |prefix| href.start_with?(prefix) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def normalize_anchor_text(text)
|
|
71
|
+
text.to_s.gsub(/\s+/, " ").strip
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def nofollow_link?(node)
|
|
75
|
+
node["rel"].to_s.split(/\s+/).any? { |value| value.casecmp?("nofollow") }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def http_internal_link?(source_url, href)
|
|
79
|
+
source_uri = URI.parse(source_url.to_s)
|
|
80
|
+
target_uri = URI.parse(URI.join(source_url, href).to_s)
|
|
81
|
+
|
|
82
|
+
source_uri.scheme == "https" && target_uri.scheme == "http" && target_uri.host == @base_host
|
|
83
|
+
rescue URI::InvalidURIError
|
|
84
|
+
false
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def normalize_internal_link(source_url, href)
|
|
88
|
+
absolute_url = URI.join(source_url, href).to_s
|
|
89
|
+
uri = URI.parse(absolute_url)
|
|
90
|
+
return if uri.host != @base_host
|
|
91
|
+
|
|
92
|
+
uri.fragment = nil
|
|
93
|
+
Url.normalize(uri.to_s, base_url: @base_url)
|
|
94
|
+
rescue URI::InvalidURIError
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class Graph
|
|
7
|
+
def initialize(links:, targets:)
|
|
8
|
+
@incoming = links.reject do |link|
|
|
9
|
+
link[:source_url] == link[:final_url] || targets.fetch(link[:target_url]).noindex?
|
|
10
|
+
end.group_by { |link| link[:final_url] }
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def inbound_count(url)
|
|
14
|
+
incoming(url).size
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def dofollow_count(url)
|
|
18
|
+
incoming(url).count { |link| !link[:nofollow] }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def nofollow_count(url)
|
|
22
|
+
incoming(url).count { |link| link[:nofollow] }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def sources(url, nofollow: nil)
|
|
26
|
+
incoming(url).filter_map do |link|
|
|
27
|
+
link[:source_url] if nofollow.nil? || link[:nofollow] == nofollow
|
|
28
|
+
end.uniq
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def incoming(url)
|
|
34
|
+
@incoming.fetch(url, [])
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class InboundChecks
|
|
7
|
+
def initialize(base_url:, graph:, scope:)
|
|
8
|
+
@base_url = base_url
|
|
9
|
+
@graph = graph
|
|
10
|
+
@scope = scope
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def call(urls:, pages:, issues:)
|
|
14
|
+
sitemap_urls = urls.filter_map do |url|
|
|
15
|
+
normalized_url = Url.normalize(url, base_url: @base_url)
|
|
16
|
+
path = Url.path(normalized_url)
|
|
17
|
+
next if path.nil?
|
|
18
|
+
next if @scope.skip_internal_path?(path)
|
|
19
|
+
|
|
20
|
+
normalized_url
|
|
21
|
+
end.to_set
|
|
22
|
+
return if sitemap_urls.size < 2
|
|
23
|
+
|
|
24
|
+
html_urls = pages.each_with_object(Set.new) do |page, result|
|
|
25
|
+
next unless page.html?
|
|
26
|
+
|
|
27
|
+
[page.normalized_url, page.normalized_final_url].compact.each do |url|
|
|
28
|
+
path = Url.path(url)
|
|
29
|
+
next if path.nil?
|
|
30
|
+
next if @scope.skip_internal_path?(path)
|
|
31
|
+
|
|
32
|
+
result << url
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
sitemap_urls.each do |target_url|
|
|
37
|
+
next unless html_urls.include?(target_url)
|
|
38
|
+
|
|
39
|
+
inbound_count = @graph.inbound_count(target_url)
|
|
40
|
+
dofollow_count = @graph.dofollow_count(target_url)
|
|
41
|
+
nofollow_count = @graph.nofollow_count(target_url)
|
|
42
|
+
|
|
43
|
+
report_orphan_page(target_url, issues) if inbound_count.zero?
|
|
44
|
+
|
|
45
|
+
if inbound_count.positive? && inbound_count < MIN_INBOUND_ANCHOR_LINKS
|
|
46
|
+
source_samples = @graph.sources(target_url)
|
|
47
|
+
|
|
48
|
+
issues.add(
|
|
49
|
+
code: :low_inbound_anchor_links,
|
|
50
|
+
severity: :warning,
|
|
51
|
+
category: :links,
|
|
52
|
+
url: target_url,
|
|
53
|
+
message: "incoming internal links: #{inbound_count}; minimum #{MIN_INBOUND_ANCHOR_LINKS}",
|
|
54
|
+
details: {inbound_count: inbound_count, minimum: MIN_INBOUND_ANCHOR_LINKS, source_urls: source_samples}
|
|
55
|
+
)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
report_low_dofollow_inlinks(target_url, dofollow_count, @graph.sources(target_url, nofollow: false), issues)
|
|
59
|
+
report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, @graph.sources(target_url, nofollow: true), issues)
|
|
60
|
+
report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, @graph.sources(target_url, nofollow: true), @graph.sources(target_url, nofollow: false), issues)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def report_orphan_page(target_url, issues)
|
|
67
|
+
issues.add(
|
|
68
|
+
code: :orphan_page,
|
|
69
|
+
severity: :warning,
|
|
70
|
+
category: :links,
|
|
71
|
+
url: target_url,
|
|
72
|
+
message: "page has no incoming internal links",
|
|
73
|
+
details: {}
|
|
74
|
+
)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def report_low_dofollow_inlinks(target_url, dofollow_count, source_samples, issues)
|
|
78
|
+
return if dofollow_count.zero?
|
|
79
|
+
return if dofollow_count >= MIN_DOFOLLOW_INBOUND_LINKS
|
|
80
|
+
|
|
81
|
+
issues.add(
|
|
82
|
+
code: :low_dofollow_inlinks,
|
|
83
|
+
severity: :warning,
|
|
84
|
+
category: :links,
|
|
85
|
+
url: target_url,
|
|
86
|
+
message: "incoming internal links without nofollow: #{dofollow_count}; minimum #{MIN_DOFOLLOW_INBOUND_LINKS}",
|
|
87
|
+
details: {dofollow_inbound_count: dofollow_count, minimum: MIN_DOFOLLOW_INBOUND_LINKS, source_urls: source_samples}
|
|
88
|
+
)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, issues)
|
|
92
|
+
return unless nofollow_count.positive? && dofollow_count.zero?
|
|
93
|
+
|
|
94
|
+
issues.add(
|
|
95
|
+
code: :only_nofollow_internal_inlinks,
|
|
96
|
+
severity: :warning,
|
|
97
|
+
category: :links,
|
|
98
|
+
url: target_url,
|
|
99
|
+
message: "all incoming internal links use nofollow",
|
|
100
|
+
details: {nofollow_inbound_count: nofollow_count, source_urls: nofollow_sources}
|
|
101
|
+
)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, dofollow_sources, issues)
|
|
105
|
+
return unless nofollow_count.positive? && dofollow_count.positive?
|
|
106
|
+
|
|
107
|
+
issues.add(
|
|
108
|
+
code: :mixed_follow_internal_inlinks,
|
|
109
|
+
severity: :notice,
|
|
110
|
+
category: :links,
|
|
111
|
+
url: target_url,
|
|
112
|
+
message: "page has nofollow and dofollow incoming internal links",
|
|
113
|
+
details: {
|
|
114
|
+
dofollow_inbound_count: dofollow_count,
|
|
115
|
+
nofollow_inbound_count: nofollow_count,
|
|
116
|
+
dofollow_source_urls: dofollow_sources,
|
|
117
|
+
nofollow_source_urls: nofollow_sources
|
|
118
|
+
}
|
|
119
|
+
)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class OutgoingChecks
|
|
7
|
+
def initialize(base_url:, targets:, allowed_statuses:, scope:)
|
|
8
|
+
@base_url = base_url
|
|
9
|
+
@targets = targets
|
|
10
|
+
@allowed_statuses = allowed_statuses
|
|
11
|
+
@scope = scope
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call(urls:, pages:, links:, resolved_links:, issues:)
|
|
15
|
+
validate_nofollow_outgoing_links(resolved_links, issues)
|
|
16
|
+
validate_http_internal_links(links, issues)
|
|
17
|
+
validate_pages_with_no_outgoing_links(urls, pages, links, issues)
|
|
18
|
+
validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def validate_nofollow_outgoing_links(links, issues)
|
|
24
|
+
links.select do |link|
|
|
25
|
+
link[:nofollow] && link[:source_url] != link[:final_url] && !@targets.fetch(link[:target_url]).noindex?
|
|
26
|
+
end.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
|
|
27
|
+
target_urls = grouped_links.map { |link| link[:target_url] }.uniq
|
|
28
|
+
|
|
29
|
+
issues.add(
|
|
30
|
+
code: :nofollow_internal_outlinks,
|
|
31
|
+
severity: :warning,
|
|
32
|
+
category: :links,
|
|
33
|
+
url: source_url,
|
|
34
|
+
message: "page has nofollow outgoing internal links",
|
|
35
|
+
details: {target_urls: target_urls}
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def validate_http_internal_links(links, issues)
|
|
41
|
+
links.select { |link| link[:http_internal_link] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
|
|
42
|
+
target_urls = grouped_links.map { |link| link[:target_url] }.uniq
|
|
43
|
+
|
|
44
|
+
issues.add(
|
|
45
|
+
code: :http_internal_link,
|
|
46
|
+
severity: :warning,
|
|
47
|
+
category: :links,
|
|
48
|
+
url: source_url,
|
|
49
|
+
message: "HTTPS page links to internal HTTP URL",
|
|
50
|
+
details: {target_urls: target_urls}
|
|
51
|
+
)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def validate_pages_with_no_outgoing_links(urls, pages, links, issues)
|
|
56
|
+
sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
|
|
57
|
+
return if sitemap_urls.size < 2
|
|
58
|
+
|
|
59
|
+
source_paths_with_links = links.map { |link| link[:source_path] }.to_set
|
|
60
|
+
|
|
61
|
+
pages.each do |page|
|
|
62
|
+
next unless page.html?
|
|
63
|
+
next unless sitemap_urls.include?(page.normalized_url)
|
|
64
|
+
|
|
65
|
+
source_path = Url.path(page.normalized_url)
|
|
66
|
+
next unless @scope.crawlable_source_path?(source_path)
|
|
67
|
+
next if source_paths_with_links.include?(source_path)
|
|
68
|
+
|
|
69
|
+
issues.add(
|
|
70
|
+
code: :page_has_no_outgoing_links,
|
|
71
|
+
severity: :warning,
|
|
72
|
+
category: :links,
|
|
73
|
+
url: page.url,
|
|
74
|
+
message: "page has no outgoing internal links",
|
|
75
|
+
details: {}
|
|
76
|
+
)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
|
|
81
|
+
sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
|
|
82
|
+
resolved_links.group_by { |link| link[:final_url] }.each do |final_url, grouped_links|
|
|
83
|
+
link = grouped_links.first
|
|
84
|
+
next if sitemap_urls.include?(final_url)
|
|
85
|
+
next unless @scope.crawlable_path?(link[:final_path])
|
|
86
|
+
|
|
87
|
+
target = @targets.fetch(final_url)
|
|
88
|
+
next unless target.allowed?(@allowed_statuses) && target.html?
|
|
89
|
+
next if target.noindex?
|
|
90
|
+
|
|
91
|
+
issues.add(
|
|
92
|
+
code: :indexable_page_missing_from_sitemap,
|
|
93
|
+
severity: :warning,
|
|
94
|
+
category: :sitemaps,
|
|
95
|
+
url: final_url,
|
|
96
|
+
message: "linked page is missing from sitemap",
|
|
97
|
+
details: {source_url: link[:source_url], source_urls: grouped_links.map { |entry| entry[:source_url] }.uniq}
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class ResolutionCheck
|
|
7
|
+
def initialize(targets:, allowed_statuses:, scope:)
|
|
8
|
+
@targets = targets
|
|
9
|
+
@allowed_statuses = allowed_statuses
|
|
10
|
+
@scope = scope
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def call(links, issues:)
|
|
14
|
+
resolved_links = []
|
|
15
|
+
grouped_links_by_target = links.group_by { |link| link[:target_url] }
|
|
16
|
+
@targets.resolve_all(grouped_links_by_target.keys)
|
|
17
|
+
|
|
18
|
+
grouped_links_by_target.each do |target_url, grouped_links|
|
|
19
|
+
target = @targets.fetch(target_url)
|
|
20
|
+
|
|
21
|
+
if target.unresolved?
|
|
22
|
+
report_unresolved_target(target_url, grouped_links, issues, target.resolution)
|
|
23
|
+
next
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if target.ignored_error?
|
|
27
|
+
next
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
unless target.allowed?(@allowed_statuses)
|
|
31
|
+
report_broken_target(target_url, grouped_links, issues, target.status)
|
|
32
|
+
next
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
report_redirect_target(target_url, grouped_links, issues, target) if target.redirect?
|
|
36
|
+
next unless @scope.crawlable_path?(target.final_path)
|
|
37
|
+
|
|
38
|
+
grouped_links.each do |link|
|
|
39
|
+
resolved_links << link.merge(final_path: target.final_path, final_url: target.final_url)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
resolved_links
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def report_broken_target(target_url, grouped_links, issues, status)
|
|
49
|
+
source_urls = grouped_links.map { |link| link[:source_url] }.uniq
|
|
50
|
+
issues.add(
|
|
51
|
+
code: :broken_internal_link,
|
|
52
|
+
severity: :warning,
|
|
53
|
+
category: :links,
|
|
54
|
+
url: target_url,
|
|
55
|
+
message: "linked URL returns HTTP #{status}",
|
|
56
|
+
details: {source_urls: source_urls, status: status}
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def report_unresolved_target(target_url, grouped_links, issues, resolution)
|
|
61
|
+
source_urls = grouped_links.map { |link| link[:source_url] }.uniq
|
|
62
|
+
suffix = (resolution && resolution[:error]) ? ": #{resolution[:error]}" : ""
|
|
63
|
+
|
|
64
|
+
issues.add(
|
|
65
|
+
code: :unresolved_internal_link,
|
|
66
|
+
severity: :warning,
|
|
67
|
+
category: :links,
|
|
68
|
+
url: target_url,
|
|
69
|
+
message: "cannot check linked URL#{suffix}",
|
|
70
|
+
details: {error: resolution && resolution[:error], source_urls: source_urls}
|
|
71
|
+
)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def report_redirect_target(target_url, grouped_links, issues, target)
|
|
75
|
+
source_urls = grouped_links.map { |link| link[:source_url] }.uniq
|
|
76
|
+
issues.add(
|
|
77
|
+
code: :internal_link_redirects,
|
|
78
|
+
severity: :warning,
|
|
79
|
+
category: :links,
|
|
80
|
+
url: target_url,
|
|
81
|
+
message: "internal link redirects to #{target.final_url}",
|
|
82
|
+
details: {final_url: target.final_url, source_urls: source_urls, status: target.status}
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class Scope
|
|
7
|
+
def crawlable_source_path?(path)
|
|
8
|
+
!path.nil? && INTERNAL_PATH_PREFIXES_TO_SKIP.none? { |prefix| path.start_with?(prefix) }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def skip_internal_path?(path)
|
|
12
|
+
return true if path == "/"
|
|
13
|
+
|
|
14
|
+
INTERNAL_PATH_PREFIXES_TO_SKIP.any? { |prefix| path.start_with?(prefix) }
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def crawlable_path?(path)
|
|
18
|
+
!path.nil? && !skip_internal_path?(path)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class TargetLookup
|
|
7
|
+
def initialize(context:)
|
|
8
|
+
@resolve_target = context.fetch(:resolve_target)
|
|
9
|
+
@resolve_targets = if context.respond_to?(:resolve_targets)
|
|
10
|
+
context.resolve_targets
|
|
11
|
+
elsif context.respond_to?(:[])
|
|
12
|
+
context[:resolve_targets]
|
|
13
|
+
end
|
|
14
|
+
@targets = {}
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def resolve_all(urls)
|
|
18
|
+
missing = urls.uniq.reject { |url| @targets.key?(url) }
|
|
19
|
+
return if missing.empty?
|
|
20
|
+
|
|
21
|
+
if @resolve_targets
|
|
22
|
+
@resolve_targets.call(missing).each do |url, resolution|
|
|
23
|
+
@targets[url] = Crawling::Target.new(target_url: url, resolution: resolution)
|
|
24
|
+
end
|
|
25
|
+
else
|
|
26
|
+
missing.each { |url| fetch(url) }
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def fetch(url)
|
|
31
|
+
@targets.fetch(url) do
|
|
32
|
+
@targets[url] = Crawling::Target.new(target_url: url, resolution: @resolve_target.call(url))
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Rules
|
|
5
|
+
class Links
|
|
6
|
+
class UrlChecks
|
|
7
|
+
def initialize(base_url:)
|
|
8
|
+
@base_url = base_url
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def call(urls:, links:, issues:)
|
|
12
|
+
checked_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }
|
|
13
|
+
checked_urls.concat(links.map { |link| link[:target_url] })
|
|
14
|
+
|
|
15
|
+
checked_urls.compact.uniq.each do |url|
|
|
16
|
+
report_url_double_slash(url, issues)
|
|
17
|
+
report_url_too_long(url, issues)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def report_url_double_slash(url, issues)
|
|
24
|
+
path = URI.parse(url).path.to_s
|
|
25
|
+
return unless path.match?(%r{//+})
|
|
26
|
+
|
|
27
|
+
issues.add(
|
|
28
|
+
code: :url_double_slash,
|
|
29
|
+
severity: :notice,
|
|
30
|
+
category: :url,
|
|
31
|
+
url: url,
|
|
32
|
+
message: "URL path contains duplicate slashes",
|
|
33
|
+
details: {path: path}
|
|
34
|
+
)
|
|
35
|
+
rescue URI::InvalidURIError
|
|
36
|
+
nil
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def report_url_too_long(url, issues)
|
|
40
|
+
return unless url.length > 2_048
|
|
41
|
+
|
|
42
|
+
issues.add(
|
|
43
|
+
code: :url_too_long,
|
|
44
|
+
severity: :notice,
|
|
45
|
+
category: :url,
|
|
46
|
+
url: url,
|
|
47
|
+
message: "URL: #{url.length} characters; maximum 2048",
|
|
48
|
+
details: {length: url.length, maximum: 2_048}
|
|
49
|
+
)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|