crawlscope 0.10.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/UPGRADE.md +76 -0
  4. data/docs/architecture.md +73 -0
  5. data/docs/rules.md +49 -0
  6. data/lib/crawlscope/configuration.rb +2 -1
  7. data/lib/crawlscope/crawl.rb +11 -79
  8. data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
  9. data/lib/crawlscope/crawling/target.rb +45 -0
  10. data/lib/crawlscope/crawling/target_resolver.rb +75 -0
  11. data/lib/crawlscope/documents/canonical.rb +22 -0
  12. data/lib/crawlscope/documents/robots.rb +57 -0
  13. data/lib/crawlscope/page.rb +4 -0
  14. data/lib/crawlscope/reporter.rb +6 -160
  15. data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
  16. data/lib/crawlscope/reporting/url_formatter.rb +29 -0
  17. data/lib/crawlscope/rule_registry.rb +2 -2
  18. data/lib/crawlscope/rules/content_quality.rb +18 -0
  19. data/lib/crawlscope/rules/indexability.rb +9 -14
  20. data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
  21. data/lib/crawlscope/rules/links/extractor.rb +100 -0
  22. data/lib/crawlscope/rules/links/graph.rb +39 -0
  23. data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
  24. data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
  25. data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
  26. data/lib/crawlscope/rules/links/scope.rb +23 -0
  27. data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
  28. data/lib/crawlscope/rules/links/url_checks.rb +54 -0
  29. data/lib/crawlscope/rules/links.rb +18 -600
  30. data/lib/crawlscope/rules/metadata.rb +2 -2
  31. data/lib/crawlscope/rules/structured_data.rb +26 -77
  32. data/lib/crawlscope/rules/uniqueness.rb +1 -1
  33. data/lib/crawlscope/schemas.rb +10 -2
  34. data/lib/crawlscope/server_timing/reporter.rb +2 -13
  35. data/lib/crawlscope/structured_data/audit.rb +17 -76
  36. data/lib/crawlscope/structured_data/check.rb +2 -0
  37. data/lib/crawlscope/structured_data/document.rb +14 -1
  38. data/lib/crawlscope/structured_data/entry.rb +31 -0
  39. data/lib/crawlscope/structured_data/outcome.rb +11 -0
  40. data/lib/crawlscope/structured_data/report.rb +6 -14
  41. data/lib/crawlscope/structured_data/reporter.rb +6 -11
  42. data/lib/crawlscope/structured_data/validator.rb +64 -0
  43. data/lib/crawlscope/version.rb +1 -1
  44. data/test/crawlscope/cli_test.rb +21 -0
  45. data/test/crawlscope/content_quality_rule_test.rb +51 -0
  46. data/test/crawlscope/links_rule_test.rb +16 -0
  47. data/test/crawlscope/reporter_test.rb +12 -0
  48. data/test/crawlscope/schema_registry_test.rb +22 -0
  49. data/test/crawlscope/structured_data_audit_test.rb +60 -0
  50. data/test/crawlscope/structured_data_report_test.rb +32 -0
  51. data/test/crawlscope/structured_data_rule_test.rb +11 -1
  52. data/test/crawlscope/structured_data_validator_test.rb +24 -0
  53. metadata +22 -1
@@ -0,0 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class Extractor
7
+ def initialize(base_url:, concurrency:, fetch_executor:, scope:)
8
+ @base_url = base_url
9
+ @base_host = URI.parse(base_url).host
10
+ @concurrency = concurrency
11
+ @fetch_executor = fetch_executor
12
+ @scope = scope
13
+ end
14
+
15
+ def call(pages)
16
+ html_pages = pages.select(&:html?)
17
+ FetchExecutor.map(
18
+ name: @fetch_executor,
19
+ concurrency: @concurrency,
20
+ items: html_pages
21
+ ) { |page| page_links(page) }.flatten
22
+ end
23
+
24
+ private
25
+
26
+ def page_links(page)
27
+ source_path = Url.path(page.normalized_url)
28
+ return [] unless @scope.crawlable_source_path?(source_path)
29
+
30
+ contextual_links(page.doc).filter_map do |node|
31
+ link_for(page: page, source_path: source_path, node: node)
32
+ end
33
+ end
34
+
35
+ def contextual_links(doc)
36
+ doc.css(LINK_SELECTORS)
37
+ end
38
+
39
+ def link_for(page:, source_path:, node:)
40
+ href = node["href"].to_s.strip
41
+ return unless crawlable_href?(href)
42
+
43
+ anchor_text = normalize_anchor_text(node.text)
44
+ return if anchor_text.empty?
45
+
46
+ target_url = normalize_internal_link(page.normalized_url, href)
47
+ return if target_url.nil?
48
+
49
+ target_path = Url.path(target_url)
50
+ return unless @scope.crawlable_path?(target_path)
51
+
52
+ {
53
+ anchor_text: anchor_text,
54
+ http_internal_link: http_internal_link?(page.normalized_url, href),
55
+ nofollow: nofollow_link?(node),
56
+ source_path: source_path,
57
+ source_url: page.normalized_url,
58
+ target_path: target_path,
59
+ target_url: target_url
60
+ }
61
+ end
62
+
63
+ def crawlable_href?(href)
64
+ return false if href.empty?
65
+ return false if href.start_with?("#")
66
+
67
+ LINK_SCHEMES_TO_SKIP.none? { |prefix| href.start_with?(prefix) }
68
+ end
69
+
70
+ def normalize_anchor_text(text)
71
+ text.to_s.gsub(/\s+/, " ").strip
72
+ end
73
+
74
+ def nofollow_link?(node)
75
+ node["rel"].to_s.split(/\s+/).any? { |value| value.casecmp?("nofollow") }
76
+ end
77
+
78
+ def http_internal_link?(source_url, href)
79
+ source_uri = URI.parse(source_url.to_s)
80
+ target_uri = URI.parse(URI.join(source_url, href).to_s)
81
+
82
+ source_uri.scheme == "https" && target_uri.scheme == "http" && target_uri.host == @base_host
83
+ rescue URI::InvalidURIError
84
+ false
85
+ end
86
+
87
+ def normalize_internal_link(source_url, href)
88
+ absolute_url = URI.join(source_url, href).to_s
89
+ uri = URI.parse(absolute_url)
90
+ return if uri.host != @base_host
91
+
92
+ uri.fragment = nil
93
+ Url.normalize(uri.to_s, base_url: @base_url)
94
+ rescue URI::InvalidURIError
95
+ nil
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class Graph
7
+ def initialize(links:, targets:)
8
+ @incoming = links.reject do |link|
9
+ link[:source_url] == link[:final_url] || targets.fetch(link[:target_url]).noindex?
10
+ end.group_by { |link| link[:final_url] }
11
+ end
12
+
13
+ def inbound_count(url)
14
+ incoming(url).size
15
+ end
16
+
17
+ def dofollow_count(url)
18
+ incoming(url).count { |link| !link[:nofollow] }
19
+ end
20
+
21
+ def nofollow_count(url)
22
+ incoming(url).count { |link| link[:nofollow] }
23
+ end
24
+
25
+ def sources(url, nofollow: nil)
26
+ incoming(url).filter_map do |link|
27
+ link[:source_url] if nofollow.nil? || link[:nofollow] == nofollow
28
+ end.uniq
29
+ end
30
+
31
+ private
32
+
33
+ def incoming(url)
34
+ @incoming.fetch(url, [])
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class InboundChecks
7
+ def initialize(base_url:, graph:, scope:)
8
+ @base_url = base_url
9
+ @graph = graph
10
+ @scope = scope
11
+ end
12
+
13
+ def call(urls:, pages:, issues:)
14
+ sitemap_urls = urls.filter_map do |url|
15
+ normalized_url = Url.normalize(url, base_url: @base_url)
16
+ path = Url.path(normalized_url)
17
+ next if path.nil?
18
+ next if @scope.skip_internal_path?(path)
19
+
20
+ normalized_url
21
+ end.to_set
22
+ return if sitemap_urls.size < 2
23
+
24
+ html_urls = pages.each_with_object(Set.new) do |page, result|
25
+ next unless page.html?
26
+
27
+ [page.normalized_url, page.normalized_final_url].compact.each do |url|
28
+ path = Url.path(url)
29
+ next if path.nil?
30
+ next if @scope.skip_internal_path?(path)
31
+
32
+ result << url
33
+ end
34
+ end
35
+
36
+ sitemap_urls.each do |target_url|
37
+ next unless html_urls.include?(target_url)
38
+
39
+ inbound_count = @graph.inbound_count(target_url)
40
+ dofollow_count = @graph.dofollow_count(target_url)
41
+ nofollow_count = @graph.nofollow_count(target_url)
42
+
43
+ report_orphan_page(target_url, issues) if inbound_count.zero?
44
+
45
+ if inbound_count.positive? && inbound_count < MIN_INBOUND_ANCHOR_LINKS
46
+ source_samples = @graph.sources(target_url)
47
+
48
+ issues.add(
49
+ code: :low_inbound_anchor_links,
50
+ severity: :warning,
51
+ category: :links,
52
+ url: target_url,
53
+ message: "incoming internal links: #{inbound_count}; minimum #{MIN_INBOUND_ANCHOR_LINKS}",
54
+ details: {inbound_count: inbound_count, minimum: MIN_INBOUND_ANCHOR_LINKS, source_urls: source_samples}
55
+ )
56
+ end
57
+
58
+ report_low_dofollow_inlinks(target_url, dofollow_count, @graph.sources(target_url, nofollow: false), issues)
59
+ report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, @graph.sources(target_url, nofollow: true), issues)
60
+ report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, @graph.sources(target_url, nofollow: true), @graph.sources(target_url, nofollow: false), issues)
61
+ end
62
+ end
63
+
64
+ private
65
+
66
+ def report_orphan_page(target_url, issues)
67
+ issues.add(
68
+ code: :orphan_page,
69
+ severity: :warning,
70
+ category: :links,
71
+ url: target_url,
72
+ message: "page has no incoming internal links",
73
+ details: {}
74
+ )
75
+ end
76
+
77
+ def report_low_dofollow_inlinks(target_url, dofollow_count, source_samples, issues)
78
+ return if dofollow_count.zero?
79
+ return if dofollow_count >= MIN_DOFOLLOW_INBOUND_LINKS
80
+
81
+ issues.add(
82
+ code: :low_dofollow_inlinks,
83
+ severity: :warning,
84
+ category: :links,
85
+ url: target_url,
86
+ message: "incoming internal links without nofollow: #{dofollow_count}; minimum #{MIN_DOFOLLOW_INBOUND_LINKS}",
87
+ details: {dofollow_inbound_count: dofollow_count, minimum: MIN_DOFOLLOW_INBOUND_LINKS, source_urls: source_samples}
88
+ )
89
+ end
90
+
91
+ def report_only_nofollow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, issues)
92
+ return unless nofollow_count.positive? && dofollow_count.zero?
93
+
94
+ issues.add(
95
+ code: :only_nofollow_internal_inlinks,
96
+ severity: :warning,
97
+ category: :links,
98
+ url: target_url,
99
+ message: "all incoming internal links use nofollow",
100
+ details: {nofollow_inbound_count: nofollow_count, source_urls: nofollow_sources}
101
+ )
102
+ end
103
+
104
+ def report_mixed_follow_internal_inlinks(target_url, nofollow_count, dofollow_count, nofollow_sources, dofollow_sources, issues)
105
+ return unless nofollow_count.positive? && dofollow_count.positive?
106
+
107
+ issues.add(
108
+ code: :mixed_follow_internal_inlinks,
109
+ severity: :notice,
110
+ category: :links,
111
+ url: target_url,
112
+ message: "page has nofollow and dofollow incoming internal links",
113
+ details: {
114
+ dofollow_inbound_count: dofollow_count,
115
+ nofollow_inbound_count: nofollow_count,
116
+ dofollow_source_urls: dofollow_sources,
117
+ nofollow_source_urls: nofollow_sources
118
+ }
119
+ )
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class OutgoingChecks
7
+ def initialize(base_url:, targets:, allowed_statuses:, scope:)
8
+ @base_url = base_url
9
+ @targets = targets
10
+ @allowed_statuses = allowed_statuses
11
+ @scope = scope
12
+ end
13
+
14
+ def call(urls:, pages:, links:, resolved_links:, issues:)
15
+ validate_nofollow_outgoing_links(resolved_links, issues)
16
+ validate_http_internal_links(links, issues)
17
+ validate_pages_with_no_outgoing_links(urls, pages, links, issues)
18
+ validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
19
+ end
20
+
21
+ private
22
+
23
+ def validate_nofollow_outgoing_links(links, issues)
24
+ links.select do |link|
25
+ link[:nofollow] && link[:source_url] != link[:final_url] && !@targets.fetch(link[:target_url]).noindex?
26
+ end.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
27
+ target_urls = grouped_links.map { |link| link[:target_url] }.uniq
28
+
29
+ issues.add(
30
+ code: :nofollow_internal_outlinks,
31
+ severity: :warning,
32
+ category: :links,
33
+ url: source_url,
34
+ message: "page has nofollow outgoing internal links",
35
+ details: {target_urls: target_urls}
36
+ )
37
+ end
38
+ end
39
+
40
+ def validate_http_internal_links(links, issues)
41
+ links.select { |link| link[:http_internal_link] }.group_by { |link| link[:source_url] }.each do |source_url, grouped_links|
42
+ target_urls = grouped_links.map { |link| link[:target_url] }.uniq
43
+
44
+ issues.add(
45
+ code: :http_internal_link,
46
+ severity: :warning,
47
+ category: :links,
48
+ url: source_url,
49
+ message: "HTTPS page links to internal HTTP URL",
50
+ details: {target_urls: target_urls}
51
+ )
52
+ end
53
+ end
54
+
55
+ def validate_pages_with_no_outgoing_links(urls, pages, links, issues)
56
+ sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
57
+ return if sitemap_urls.size < 2
58
+
59
+ source_paths_with_links = links.map { |link| link[:source_path] }.to_set
60
+
61
+ pages.each do |page|
62
+ next unless page.html?
63
+ next unless sitemap_urls.include?(page.normalized_url)
64
+
65
+ source_path = Url.path(page.normalized_url)
66
+ next unless @scope.crawlable_source_path?(source_path)
67
+ next if source_paths_with_links.include?(source_path)
68
+
69
+ issues.add(
70
+ code: :page_has_no_outgoing_links,
71
+ severity: :warning,
72
+ category: :links,
73
+ url: page.url,
74
+ message: "page has no outgoing internal links",
75
+ details: {}
76
+ )
77
+ end
78
+ end
79
+
80
+ def validate_indexable_pages_missing_from_sitemap(urls, resolved_links, issues)
81
+ sitemap_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }.compact.to_set
82
+ resolved_links.group_by { |link| link[:final_url] }.each do |final_url, grouped_links|
83
+ link = grouped_links.first
84
+ next if sitemap_urls.include?(final_url)
85
+ next unless @scope.crawlable_path?(link[:final_path])
86
+
87
+ target = @targets.fetch(final_url)
88
+ next unless target.allowed?(@allowed_statuses) && target.html?
89
+ next if target.noindex?
90
+
91
+ issues.add(
92
+ code: :indexable_page_missing_from_sitemap,
93
+ severity: :warning,
94
+ category: :sitemaps,
95
+ url: final_url,
96
+ message: "linked page is missing from sitemap",
97
+ details: {source_url: link[:source_url], source_urls: grouped_links.map { |entry| entry[:source_url] }.uniq}
98
+ )
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,88 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class ResolutionCheck
7
+ def initialize(targets:, allowed_statuses:, scope:)
8
+ @targets = targets
9
+ @allowed_statuses = allowed_statuses
10
+ @scope = scope
11
+ end
12
+
13
+ def call(links, issues:)
14
+ resolved_links = []
15
+ grouped_links_by_target = links.group_by { |link| link[:target_url] }
16
+ @targets.resolve_all(grouped_links_by_target.keys)
17
+
18
+ grouped_links_by_target.each do |target_url, grouped_links|
19
+ target = @targets.fetch(target_url)
20
+
21
+ if target.unresolved?
22
+ report_unresolved_target(target_url, grouped_links, issues, target.resolution)
23
+ next
24
+ end
25
+
26
+ if target.ignored_error?
27
+ next
28
+ end
29
+
30
+ unless target.allowed?(@allowed_statuses)
31
+ report_broken_target(target_url, grouped_links, issues, target.status)
32
+ next
33
+ end
34
+
35
+ report_redirect_target(target_url, grouped_links, issues, target) if target.redirect?
36
+ next unless @scope.crawlable_path?(target.final_path)
37
+
38
+ grouped_links.each do |link|
39
+ resolved_links << link.merge(final_path: target.final_path, final_url: target.final_url)
40
+ end
41
+ end
42
+
43
+ resolved_links
44
+ end
45
+
46
+ private
47
+
48
+ def report_broken_target(target_url, grouped_links, issues, status)
49
+ source_urls = grouped_links.map { |link| link[:source_url] }.uniq
50
+ issues.add(
51
+ code: :broken_internal_link,
52
+ severity: :warning,
53
+ category: :links,
54
+ url: target_url,
55
+ message: "linked URL returns HTTP #{status}",
56
+ details: {source_urls: source_urls, status: status}
57
+ )
58
+ end
59
+
60
+ def report_unresolved_target(target_url, grouped_links, issues, resolution)
61
+ source_urls = grouped_links.map { |link| link[:source_url] }.uniq
62
+ suffix = (resolution && resolution[:error]) ? ": #{resolution[:error]}" : ""
63
+
64
+ issues.add(
65
+ code: :unresolved_internal_link,
66
+ severity: :warning,
67
+ category: :links,
68
+ url: target_url,
69
+ message: "cannot check linked URL#{suffix}",
70
+ details: {error: resolution && resolution[:error], source_urls: source_urls}
71
+ )
72
+ end
73
+
74
+ def report_redirect_target(target_url, grouped_links, issues, target)
75
+ source_urls = grouped_links.map { |link| link[:source_url] }.uniq
76
+ issues.add(
77
+ code: :internal_link_redirects,
78
+ severity: :warning,
79
+ category: :links,
80
+ url: target_url,
81
+ message: "internal link redirects to #{target.final_url}",
82
+ details: {final_url: target.final_url, source_urls: source_urls, status: target.status}
83
+ )
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class Scope
7
+ def crawlable_source_path?(path)
8
+ !path.nil? && INTERNAL_PATH_PREFIXES_TO_SKIP.none? { |prefix| path.start_with?(prefix) }
9
+ end
10
+
11
+ def skip_internal_path?(path)
12
+ return true if path == "/"
13
+
14
+ INTERNAL_PATH_PREFIXES_TO_SKIP.any? { |prefix| path.start_with?(prefix) }
15
+ end
16
+
17
+ def crawlable_path?(path)
18
+ !path.nil? && !skip_internal_path?(path)
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class TargetLookup
7
+ def initialize(context:)
8
+ @resolve_target = context.fetch(:resolve_target)
9
+ @resolve_targets = if context.respond_to?(:resolve_targets)
10
+ context.resolve_targets
11
+ elsif context.respond_to?(:[])
12
+ context[:resolve_targets]
13
+ end
14
+ @targets = {}
15
+ end
16
+
17
+ def resolve_all(urls)
18
+ missing = urls.uniq.reject { |url| @targets.key?(url) }
19
+ return if missing.empty?
20
+
21
+ if @resolve_targets
22
+ @resolve_targets.call(missing).each do |url, resolution|
23
+ @targets[url] = Crawling::Target.new(target_url: url, resolution: resolution)
24
+ end
25
+ else
26
+ missing.each { |url| fetch(url) }
27
+ end
28
+ end
29
+
30
+ def fetch(url)
31
+ @targets.fetch(url) do
32
+ @targets[url] = Crawling::Target.new(target_url: url, resolution: @resolve_target.call(url))
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Rules
5
+ class Links
6
+ class UrlChecks
7
+ def initialize(base_url:)
8
+ @base_url = base_url
9
+ end
10
+
11
+ def call(urls:, links:, issues:)
12
+ checked_urls = urls.map { |url| Url.normalize(url, base_url: @base_url) }
13
+ checked_urls.concat(links.map { |link| link[:target_url] })
14
+
15
+ checked_urls.compact.uniq.each do |url|
16
+ report_url_double_slash(url, issues)
17
+ report_url_too_long(url, issues)
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ def report_url_double_slash(url, issues)
24
+ path = URI.parse(url).path.to_s
25
+ return unless path.match?(%r{//+})
26
+
27
+ issues.add(
28
+ code: :url_double_slash,
29
+ severity: :notice,
30
+ category: :url,
31
+ url: url,
32
+ message: "URL path contains duplicate slashes",
33
+ details: {path: path}
34
+ )
35
+ rescue URI::InvalidURIError
36
+ nil
37
+ end
38
+
39
+ def report_url_too_long(url, issues)
40
+ return unless url.length > 2_048
41
+
42
+ issues.add(
43
+ code: :url_too_long,
44
+ severity: :notice,
45
+ category: :url,
46
+ url: url,
47
+ message: "URL: #{url.length} characters; maximum 2048",
48
+ details: {length: url.length, maximum: 2_048}
49
+ )
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end