crawlscope 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/UPGRADE.md +76 -0
- data/docs/architecture.md +73 -0
- data/docs/rules.md +49 -0
- data/lib/crawlscope/configuration.rb +2 -1
- data/lib/crawlscope/crawl.rb +11 -79
- data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
- data/lib/crawlscope/crawling/target.rb +45 -0
- data/lib/crawlscope/crawling/target_resolver.rb +75 -0
- data/lib/crawlscope/documents/canonical.rb +22 -0
- data/lib/crawlscope/documents/robots.rb +57 -0
- data/lib/crawlscope/page.rb +4 -0
- data/lib/crawlscope/reporter.rb +6 -160
- data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
- data/lib/crawlscope/reporting/url_formatter.rb +29 -0
- data/lib/crawlscope/rule_registry.rb +2 -2
- data/lib/crawlscope/rules/content_quality.rb +18 -0
- data/lib/crawlscope/rules/indexability.rb +9 -14
- data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
- data/lib/crawlscope/rules/links/extractor.rb +100 -0
- data/lib/crawlscope/rules/links/graph.rb +39 -0
- data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
- data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
- data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
- data/lib/crawlscope/rules/links/scope.rb +23 -0
- data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
- data/lib/crawlscope/rules/links/url_checks.rb +54 -0
- data/lib/crawlscope/rules/links.rb +18 -600
- data/lib/crawlscope/rules/metadata.rb +2 -2
- data/lib/crawlscope/rules/structured_data.rb +26 -77
- data/lib/crawlscope/rules/uniqueness.rb +1 -1
- data/lib/crawlscope/schemas.rb +10 -2
- data/lib/crawlscope/server_timing/reporter.rb +2 -13
- data/lib/crawlscope/structured_data/audit.rb +17 -76
- data/lib/crawlscope/structured_data/check.rb +2 -0
- data/lib/crawlscope/structured_data/document.rb +14 -1
- data/lib/crawlscope/structured_data/entry.rb +31 -0
- data/lib/crawlscope/structured_data/outcome.rb +11 -0
- data/lib/crawlscope/structured_data/report.rb +6 -14
- data/lib/crawlscope/structured_data/reporter.rb +6 -11
- data/lib/crawlscope/structured_data/validator.rb +64 -0
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/cli_test.rb +21 -0
- data/test/crawlscope/content_quality_rule_test.rb +51 -0
- data/test/crawlscope/links_rule_test.rb +16 -0
- data/test/crawlscope/reporter_test.rb +12 -0
- data/test/crawlscope/schema_registry_test.rb +22 -0
- data/test/crawlscope/structured_data_audit_test.rb +60 -0
- data/test/crawlscope/structured_data_report_test.rb +32 -0
- data/test/crawlscope/structured_data_rule_test.rb +11 -1
- data/test/crawlscope/structured_data_validator_test.rb +24 -0
- metadata +22 -1
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
module Crawlscope
|
|
4
4
|
module Rules
|
|
5
5
|
class StructuredData
|
|
6
|
-
CAREER_DETAIL_PATH = %r{/careers/[^/]+/?\z}
|
|
7
|
-
|
|
8
6
|
attr_reader :code
|
|
9
7
|
|
|
10
|
-
def initialize
|
|
8
|
+
def initialize(job_posting_path: nil, required_types: nil)
|
|
9
|
+
@job_posting_path = job_posting_path
|
|
10
|
+
@required_types = required_types
|
|
11
11
|
@code = :structured_data
|
|
12
12
|
end
|
|
13
13
|
|
|
@@ -24,8 +24,10 @@ module Crawlscope
|
|
|
24
24
|
private
|
|
25
25
|
|
|
26
26
|
def validate_page(page, issues, schema_registry)
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
validation = Crawlscope::StructuredData::Validator.new(schema_registry: schema_registry).call(
|
|
28
|
+
html: page.body, required_types: @required_types ? @required_types.call(page.url) : []
|
|
29
|
+
)
|
|
30
|
+
items = validation.items
|
|
29
31
|
|
|
30
32
|
if items.empty?
|
|
31
33
|
issues.add(
|
|
@@ -39,38 +41,26 @@ module Crawlscope
|
|
|
39
41
|
return
|
|
40
42
|
end
|
|
41
43
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
44
|
+
validation.findings.each do |finding|
|
|
45
|
+
details = {source: finding.source}
|
|
46
|
+
message = case finding.code
|
|
47
|
+
when :structured_data_parse_error
|
|
48
|
+
"#{source_label(finding.source)} parse error: #{finding.errors.first[:issue]}"
|
|
49
|
+
when :structured_data_missing_type
|
|
50
|
+
details[:paths] = finding.paths
|
|
51
|
+
"#{source_label(finding.source)} structured data missing @type"
|
|
52
|
+
when :structured_data_missing_required_type
|
|
53
|
+
details[:expected_types] = finding.errors.first.fetch(:expected_types)
|
|
54
|
+
"page missing required structured data: #{finding.type}"
|
|
55
|
+
else
|
|
56
|
+
details[:errors] = finding.errors
|
|
57
|
+
schema_error_message(finding.source, finding.errors)
|
|
56
58
|
end
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
errors = schema_registry.validate(data)
|
|
61
|
-
next if errors.empty?
|
|
62
|
-
|
|
63
|
-
issues.add(
|
|
64
|
-
code: :structured_data_schema_error,
|
|
65
|
-
severity: :warning,
|
|
66
|
-
category: :structured_data,
|
|
67
|
-
url: page.url,
|
|
68
|
-
message: schema_error_message(source, errors),
|
|
69
|
-
details: {errors: errors, source: source}
|
|
70
|
-
)
|
|
59
|
+
issues.add(code: finding.code, severity: :warning, category: :structured_data,
|
|
60
|
+
url: page.url, message: message, details: details)
|
|
71
61
|
end
|
|
72
62
|
|
|
73
|
-
validate_job_posting_count(page, items, issues)
|
|
63
|
+
validate_job_posting_count(page, items, issues) if @job_posting_path && career_detail_page?(page.url)
|
|
74
64
|
end
|
|
75
65
|
|
|
76
66
|
def source_label(source)
|
|
@@ -83,7 +73,7 @@ module Crawlscope
|
|
|
83
73
|
end
|
|
84
74
|
|
|
85
75
|
def validate_job_posting_count(page, items, issues)
|
|
86
|
-
job_postings = items.select { |item|
|
|
76
|
+
job_postings = items.select { |item| item.types.include?("JobPosting") }
|
|
87
77
|
return if job_postings.size == 1
|
|
88
78
|
|
|
89
79
|
if job_postings.size > 1
|
|
@@ -107,49 +97,8 @@ module Crawlscope
|
|
|
107
97
|
end
|
|
108
98
|
end
|
|
109
99
|
|
|
110
|
-
def validate_type_presence(page, source, data, issues)
|
|
111
|
-
missing_paths = missing_type_paths(data)
|
|
112
|
-
return if missing_paths.empty?
|
|
113
|
-
|
|
114
|
-
issues.add(
|
|
115
|
-
code: :structured_data_missing_type,
|
|
116
|
-
severity: :warning,
|
|
117
|
-
category: :structured_data,
|
|
118
|
-
url: page.url,
|
|
119
|
-
message: "#{source_label(source)} structured data missing @type",
|
|
120
|
-
details: {paths: missing_paths, source: source}
|
|
121
|
-
)
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
def missing_type_paths(data, path = "$")
|
|
125
|
-
return [] unless data.is_a?(Hash)
|
|
126
|
-
|
|
127
|
-
paths = []
|
|
128
|
-
paths << path if data["@type"].to_s.strip.empty?
|
|
129
|
-
|
|
130
|
-
if data["@graph"].is_a?(Array)
|
|
131
|
-
data["@graph"].each_with_index do |entry, index|
|
|
132
|
-
paths.concat(missing_type_paths(entry, "#{path}.@graph[#{index}]"))
|
|
133
|
-
end
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
paths
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
def structured_data_types(data)
|
|
140
|
-
return [] unless data.is_a?(Hash)
|
|
141
|
-
|
|
142
|
-
types = Array(data["@type"]).map(&:to_s)
|
|
143
|
-
|
|
144
|
-
if data["@graph"].is_a?(Array)
|
|
145
|
-
types.concat(data["@graph"].flat_map { |entry| structured_data_types(entry) })
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
types
|
|
149
|
-
end
|
|
150
|
-
|
|
151
100
|
def career_detail_page?(url)
|
|
152
|
-
URI(url).path.match?(
|
|
101
|
+
URI(url).path.match?(@job_posting_path)
|
|
153
102
|
rescue URI::InvalidURIError
|
|
154
103
|
false
|
|
155
104
|
end
|
|
@@ -57,7 +57,7 @@ module Crawlscope
|
|
|
57
57
|
|
|
58
58
|
{
|
|
59
59
|
content_fingerprint_digest: content_fingerprint_digest(page.doc),
|
|
60
|
-
canonical: page.
|
|
60
|
+
canonical: page.canonical.href,
|
|
61
61
|
description: page.doc.at_css('meta[name="description"]')&.[]("content").to_s.strip,
|
|
62
62
|
shingles: shingles_for(tokens),
|
|
63
63
|
title: page.doc.at_css("title")&.text.to_s.strip,
|
data/lib/crawlscope/schemas.rb
CHANGED
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
module Crawlscope
|
|
4
4
|
class Schemas
|
|
5
|
+
# Draft 4 omits date; the native Draft 3 format checks real calendar dates.
|
|
6
|
+
DATE_OR_DATE_TIME = {
|
|
7
|
+
anyOf: [
|
|
8
|
+
{type: "string", format: "date-time"},
|
|
9
|
+
{"$schema" => "http://json-schema.org/draft-03/schema#", "type" => "string", "format" => "date"}
|
|
10
|
+
]
|
|
11
|
+
}.freeze
|
|
12
|
+
|
|
5
13
|
FAQ_PAGE = {
|
|
6
14
|
"type" => "object",
|
|
7
15
|
"required" => ["@context", "@type", "mainEntity"],
|
|
@@ -123,7 +131,7 @@ module Crawlscope
|
|
|
123
131
|
:itemReviewed => {type: "object"},
|
|
124
132
|
:reviewRating => RATING,
|
|
125
133
|
:author => {type: ["object", "string"]},
|
|
126
|
-
:datePublished =>
|
|
134
|
+
:datePublished => DATE_OR_DATE_TIME,
|
|
127
135
|
:reviewBody => {type: "string"}
|
|
128
136
|
}
|
|
129
137
|
}.freeze
|
|
@@ -136,7 +144,7 @@ module Crawlscope
|
|
|
136
144
|
:reviewRating => RATING,
|
|
137
145
|
:author => {type: ["object", "string"]},
|
|
138
146
|
:reviewBody => {type: "string"},
|
|
139
|
-
:datePublished =>
|
|
147
|
+
:datePublished => DATE_OR_DATE_TIME
|
|
140
148
|
}
|
|
141
149
|
}.freeze
|
|
142
150
|
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "uri"
|
|
4
|
-
|
|
5
3
|
module Crawlscope
|
|
6
4
|
class ServerTiming
|
|
7
5
|
class Reporter
|
|
@@ -71,11 +69,12 @@ module Crawlscope
|
|
|
71
69
|
return if samples.empty?
|
|
72
70
|
|
|
73
71
|
@io.puts(" Pages above #{SLOW_PAGE_THRESHOLD_MS} ms:")
|
|
72
|
+
urls = Reporting::UrlFormatter.new(base_url: base_url)
|
|
74
73
|
samples.each do |sample|
|
|
75
74
|
metric = sample[:metric]
|
|
76
75
|
detail = metric.description ? " #{description(metric.description)}" : ""
|
|
77
76
|
@io.puts(
|
|
78
|
-
" #{
|
|
77
|
+
" #{urls.call(sample[:page].url)}: " \
|
|
79
78
|
"#{duration(metric.duration)} (#{metric.name}#{detail})"
|
|
80
79
|
)
|
|
81
80
|
end
|
|
@@ -96,16 +95,6 @@ module Crawlscope
|
|
|
96
95
|
format("%.1f%%", numerator.fdiv(denominator) * 100)
|
|
97
96
|
end
|
|
98
97
|
|
|
99
|
-
def relative_url(url, base_url:)
|
|
100
|
-
uri = URI.parse(url)
|
|
101
|
-
base_uri = URI.parse(base_url)
|
|
102
|
-
return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
|
|
103
|
-
|
|
104
|
-
uri.request_uri
|
|
105
|
-
rescue URI::InvalidURIError
|
|
106
|
-
url
|
|
107
|
-
end
|
|
108
|
-
|
|
109
98
|
def pluralize(word, count)
|
|
110
99
|
(count == 1) ? word.to_s : "#{word}s"
|
|
111
100
|
end
|
|
@@ -3,37 +3,17 @@
|
|
|
3
3
|
module Crawlscope
|
|
4
4
|
module StructuredData
|
|
5
5
|
class Audit
|
|
6
|
-
Page =
|
|
7
|
-
|
|
8
|
-
structured_items.count { |item| item[:source] == "json-ld" }
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def microdata_count
|
|
12
|
-
structured_items.count { |item| item[:source] == "microdata" }
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def ok?
|
|
16
|
-
fetch_error.nil? && errors.empty?
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def structured_data_found?
|
|
20
|
-
skipped_reason == "non-html" || structured_items.any?
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
Outcome = Data.define(:entries) do
|
|
25
|
-
def ok?
|
|
26
|
-
entries.all?(&:ok?)
|
|
27
|
-
end
|
|
28
|
-
end
|
|
6
|
+
Page = Entry
|
|
7
|
+
Outcome = Crawlscope::StructuredData::Outcome
|
|
29
8
|
|
|
30
|
-
def initialize(schema_registry:, renderer:, timeout_seconds:, browser_factory: nil, network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS, scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE)
|
|
9
|
+
def initialize(schema_registry:, renderer:, timeout_seconds:, browser_factory: nil, network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS, scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE, required_types: nil)
|
|
31
10
|
@schema_registry = schema_registry
|
|
32
11
|
@renderer = renderer.to_sym
|
|
33
12
|
@timeout_seconds = timeout_seconds
|
|
34
13
|
@browser_factory = browser_factory
|
|
35
14
|
@network_idle_timeout_seconds = network_idle_timeout_seconds
|
|
36
15
|
@scroll_page = scroll_page
|
|
16
|
+
@required_types = required_types
|
|
37
17
|
end
|
|
38
18
|
|
|
39
19
|
def call(urls:)
|
|
@@ -47,68 +27,29 @@ module Crawlscope
|
|
|
47
27
|
|
|
48
28
|
private
|
|
49
29
|
|
|
50
|
-
def build_browser(base_url)
|
|
51
|
-
browser_factory = @browser_factory
|
|
52
|
-
|
|
53
|
-
if browser_factory
|
|
54
|
-
browser_factory.call
|
|
55
|
-
else
|
|
56
|
-
Crawlscope::Browser.new(
|
|
57
|
-
base_url: base_url,
|
|
58
|
-
timeout_seconds: @timeout_seconds,
|
|
59
|
-
network_idle_timeout_seconds: @network_idle_timeout_seconds,
|
|
60
|
-
scroll_page: @scroll_page
|
|
61
|
-
)
|
|
62
|
-
end
|
|
63
|
-
rescue LoadError => error
|
|
64
|
-
raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
|
|
65
|
-
end
|
|
66
|
-
|
|
67
30
|
def build_fetcher(urls)
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
Crawlscope::Http.new(base_url: base_url, timeout_seconds: @timeout_seconds)
|
|
75
|
-
end
|
|
31
|
+
Crawling::FetcherFactory.new(
|
|
32
|
+
base_url: urls.first || "http://localhost:3000", renderer: @renderer,
|
|
33
|
+
timeout_seconds: @timeout_seconds, browser_factory: @browser_factory,
|
|
34
|
+
network_idle_timeout_seconds: @network_idle_timeout_seconds,
|
|
35
|
+
scroll_page: @scroll_page
|
|
36
|
+
).call
|
|
76
37
|
end
|
|
77
38
|
|
|
78
39
|
def build_validation_errors(page)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
}
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
errors = structured_items.filter_map do |item|
|
|
88
|
-
if item[:data].is_a?(Hash) && item[:data][:error]
|
|
89
|
-
{
|
|
90
|
-
errors: [{field: "parse", issue: item[:data][:message]}],
|
|
91
|
-
source: item[:source],
|
|
92
|
-
type: item[:source]
|
|
93
|
-
}
|
|
94
|
-
else
|
|
95
|
-
schema_errors = @schema_registry.validate(item[:data])
|
|
96
|
-
next if schema_errors.empty?
|
|
97
|
-
|
|
98
|
-
{
|
|
99
|
-
errors: schema_errors,
|
|
100
|
-
source: item[:source],
|
|
101
|
-
type: item[:data]["@type"] || item[:source]
|
|
102
|
-
}
|
|
103
|
-
end
|
|
40
|
+
validation = Validator.new(schema_registry: @schema_registry).call(
|
|
41
|
+
html: page.body, required_types: @required_types ? @required_types.call(page.url) : []
|
|
42
|
+
)
|
|
43
|
+
structured_items = validation.items.map { |item| {data: item.data, source: item.source} }
|
|
44
|
+
errors = validation.findings.map do |finding|
|
|
45
|
+
{errors: finding.errors, source: finding.source, type: finding.type}
|
|
104
46
|
end
|
|
105
|
-
|
|
106
47
|
[structured_items, errors]
|
|
107
48
|
end
|
|
108
49
|
|
|
109
50
|
def validate_url(url, fetcher)
|
|
110
51
|
page = fetcher.fetch(url)
|
|
111
|
-
content_type = page.
|
|
52
|
+
content_type = page.header("content-type").to_s
|
|
112
53
|
|
|
113
54
|
if page.error
|
|
114
55
|
Page.new(url: url, status: page.status, structured_items: [], errors: [], fetch_error: page.error, content_type: content_type, skipped_reason: nil)
|
|
@@ -5,6 +5,7 @@ module Crawlscope
|
|
|
5
5
|
class Check
|
|
6
6
|
def initialize(configuration:)
|
|
7
7
|
@configuration = configuration
|
|
8
|
+
@required_types = configuration.required_structured_data_types if configuration.respond_to?(:required_structured_data_types)
|
|
8
9
|
end
|
|
9
10
|
|
|
10
11
|
def call(urls:, debug: false, renderer: @configuration.renderer, timeout_seconds: @configuration.timeout_seconds, report_path: nil, summary: false)
|
|
@@ -25,6 +26,7 @@ module Crawlscope
|
|
|
25
26
|
browser_factory: @configuration.browser_factory,
|
|
26
27
|
network_idle_timeout_seconds: @configuration.network_idle_timeout_seconds,
|
|
27
28
|
renderer: renderer,
|
|
29
|
+
required_types: @required_types,
|
|
28
30
|
schema_registry: @configuration.schema_registry,
|
|
29
31
|
scroll_page: @configuration.scroll_page?,
|
|
30
32
|
timeout_seconds: timeout_seconds
|
|
@@ -6,7 +6,20 @@ require "nokogiri"
|
|
|
6
6
|
module Crawlscope
|
|
7
7
|
module StructuredData
|
|
8
8
|
class Document
|
|
9
|
-
Item = Data.define(:source, :data)
|
|
9
|
+
Item = Data.define(:source, :data) do
|
|
10
|
+
def types
|
|
11
|
+
nodes = [data]
|
|
12
|
+
types = []
|
|
13
|
+
until nodes.empty?
|
|
14
|
+
node = nodes.pop
|
|
15
|
+
next unless node.is_a?(Hash)
|
|
16
|
+
|
|
17
|
+
types.concat(Array(node["@type"]))
|
|
18
|
+
nodes.concat(Array(node["@graph"]))
|
|
19
|
+
end
|
|
20
|
+
types
|
|
21
|
+
end
|
|
22
|
+
end
|
|
10
23
|
|
|
11
24
|
def initialize(html:)
|
|
12
25
|
@doc = Nokogiri::HTML(html.to_s)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module StructuredData
|
|
5
|
+
Entry = Data.define(:url, :status, :structured_items, :errors, :fetch_error, :content_type, :skipped_reason) do
|
|
6
|
+
def json_ld_count
|
|
7
|
+
structured_items.count { |item| item[:source] == "json-ld" }
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def microdata_count
|
|
11
|
+
structured_items.count { |item| item[:source] == "microdata" }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def ok?
|
|
15
|
+
failure_kind.nil?
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def failure_kind
|
|
19
|
+
return :http if fetch_error
|
|
20
|
+
return :missing_data unless structured_data_found?
|
|
21
|
+
return :validation if errors.any?
|
|
22
|
+
|
|
23
|
+
nil
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def structured_data_found?
|
|
27
|
+
skipped_reason == "non-html" || structured_items.any?
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -8,19 +8,19 @@ module Crawlscope
|
|
|
8
8
|
end
|
|
9
9
|
|
|
10
10
|
def all_valid?
|
|
11
|
-
|
|
11
|
+
@result.ok?
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
def failure_count
|
|
15
|
-
|
|
15
|
+
entries.count { |entry| !entry.ok? }
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def http_errors
|
|
19
|
-
entries.select { |entry| entry.
|
|
19
|
+
entries.select { |entry| entry.failure_kind == :http }
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def missing_data
|
|
23
|
-
entries.select { |entry| entry.
|
|
23
|
+
entries.select { |entry| entry.failure_kind == :missing_data }
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
def results
|
|
@@ -34,7 +34,7 @@ module Crawlscope
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
def validation_errors
|
|
37
|
-
entries.select { |entry| entry.
|
|
37
|
+
entries.select { |entry| entry.failure_kind == :validation }
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
private
|
|
@@ -44,15 +44,7 @@ module Crawlscope
|
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
def result_for(entry)
|
|
47
|
-
if entry.fetch_error
|
|
48
|
-
{
|
|
49
|
-
status: entry.status,
|
|
50
|
-
error: entry.fetch_error,
|
|
51
|
-
structured_data_found: false,
|
|
52
|
-
validation_errors: [],
|
|
53
|
-
json_ld_count: 0
|
|
54
|
-
}
|
|
55
|
-
elsif entry.fetch_error
|
|
47
|
+
if entry.fetch_error
|
|
56
48
|
{
|
|
57
49
|
status: entry.status || "exception",
|
|
58
50
|
error: entry.fetch_error,
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "uri"
|
|
4
3
|
require "json"
|
|
5
4
|
|
|
6
5
|
module Crawlscope
|
|
@@ -36,12 +35,6 @@ module Crawlscope
|
|
|
36
35
|
|
|
37
36
|
private
|
|
38
37
|
|
|
39
|
-
def extract_path(url)
|
|
40
|
-
URI.parse(url).path
|
|
41
|
-
rescue URI::InvalidURIError
|
|
42
|
-
url
|
|
43
|
-
end
|
|
44
|
-
|
|
45
38
|
def print_category(name, items)
|
|
46
39
|
return if items.empty?
|
|
47
40
|
|
|
@@ -56,15 +49,15 @@ module Crawlscope
|
|
|
56
49
|
@io.puts("")
|
|
57
50
|
|
|
58
51
|
print_category("HTTP ERRORS", report.http_errors) do |entry|
|
|
59
|
-
@io.puts("• #{
|
|
52
|
+
@io.puts("• #{entry.url} (#{entry.status}: #{entry.fetch_error})")
|
|
60
53
|
end
|
|
61
54
|
|
|
62
55
|
print_category("MISSING STRUCTURED DATA", report.missing_data) do |entry|
|
|
63
|
-
@io.puts("• #{
|
|
56
|
+
@io.puts("• #{entry.url}")
|
|
64
57
|
end
|
|
65
58
|
|
|
66
59
|
print_category("VALIDATION ERRORS", report.validation_errors) do |entry|
|
|
67
|
-
@io.puts("• #{
|
|
60
|
+
@io.puts("• #{entry.url}")
|
|
68
61
|
|
|
69
62
|
entry.errors.each do |error|
|
|
70
63
|
@io.puts(" #{source_label(error[:source])}:")
|
|
@@ -121,7 +114,9 @@ module Crawlscope
|
|
|
121
114
|
|
|
122
115
|
def report_validation(entry)
|
|
123
116
|
@io.puts("")
|
|
124
|
-
if entry.
|
|
117
|
+
if !entry.structured_data_found?
|
|
118
|
+
@io.puts("Validation: FAILED (no structured data found)")
|
|
119
|
+
elsif entry.errors.empty?
|
|
125
120
|
@io.puts("Validation: OK")
|
|
126
121
|
else
|
|
127
122
|
@io.puts("Validation: FAILED")
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module StructuredData
|
|
5
|
+
class Validator
|
|
6
|
+
Finding = Data.define(:code, :source, :type, :errors, :paths)
|
|
7
|
+
Validation = Data.define(:items, :findings)
|
|
8
|
+
|
|
9
|
+
def initialize(schema_registry:)
|
|
10
|
+
@schema_registry = schema_registry
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def call(html:, required_types: [])
|
|
14
|
+
items = Document.new(html: html).items
|
|
15
|
+
findings = items.flat_map { |item| validate(item) }
|
|
16
|
+
types = items.flat_map(&:types)
|
|
17
|
+
Array(required_types).each do |requirement|
|
|
18
|
+
alternatives = Array(requirement)
|
|
19
|
+
next if (alternatives & types).any?
|
|
20
|
+
|
|
21
|
+
expected = alternatives.join(" or ")
|
|
22
|
+
findings << Finding.new(code: :structured_data_missing_required_type, source: "page", type: expected,
|
|
23
|
+
errors: [{field: "@type", issue: "must include #{expected}", expected_types: alternatives}], paths: [])
|
|
24
|
+
end
|
|
25
|
+
Validation.new(items: items, findings: findings)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def validate(item)
|
|
31
|
+
data = item.data
|
|
32
|
+
if data.is_a?(Hash) && data[:error]
|
|
33
|
+
return [finding(:structured_data_parse_error, item, [{field: "parse", issue: data[:message]}])]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
findings = []
|
|
37
|
+
paths = missing_type_paths(data)
|
|
38
|
+
unless paths.empty?
|
|
39
|
+
errors = paths.map { |path| {field: "#{path}.@type", issue: "is required"} }
|
|
40
|
+
findings << finding(:structured_data_missing_type, item, errors, paths)
|
|
41
|
+
end
|
|
42
|
+
errors = @schema_registry.validate(data)
|
|
43
|
+
findings << finding(:structured_data_schema_error, item, errors) unless errors.empty?
|
|
44
|
+
findings
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def finding(code, item, errors, paths = [])
|
|
48
|
+
Finding.new(code: code, source: item.source, type: item.data["@type"] || item.source, errors: errors, paths: paths)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def missing_type_paths(data, path = "$")
|
|
52
|
+
return [] unless data.is_a?(Hash)
|
|
53
|
+
|
|
54
|
+
paths = data["@type"].to_s.strip.empty? ? [path] : []
|
|
55
|
+
if data["@graph"].is_a?(Array)
|
|
56
|
+
data["@graph"].each_with_index do |entry, index|
|
|
57
|
+
paths.concat(missing_type_paths(entry, "#{path}.@graph[#{index}]"))
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
paths
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/crawlscope/version.rb
CHANGED
data/test/crawlscope/cli_test.rb
CHANGED
|
@@ -3,6 +3,27 @@
|
|
|
3
3
|
require "test_helper"
|
|
4
4
|
|
|
5
5
|
class CrawlscopeCliTest < Minitest::Test
|
|
6
|
+
def test_ldjson_missing_data_fails_in_details_summary_json_and_exit_status
|
|
7
|
+
url = "https://example.com/page?locale=en"
|
|
8
|
+
stub_request(:get, url).to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<h1>No markup</h1>")
|
|
9
|
+
out = StringIO.new
|
|
10
|
+
err = StringIO.new
|
|
11
|
+
|
|
12
|
+
Dir.mktmpdir do |directory|
|
|
13
|
+
path = File.join(directory, "report.json")
|
|
14
|
+
status = Crawlscope::Cli.start(["ldjson", "--url", url, "--summary", "--report-path", path], out: out, err: err)
|
|
15
|
+
|
|
16
|
+
assert_equal 1, status
|
|
17
|
+
assert_empty err.string
|
|
18
|
+
assert_includes out.string, "Validation: FAILED (no structured data found)"
|
|
19
|
+
assert_includes out.string, "STATUS: FAILED"
|
|
20
|
+
assert_includes out.string, "MISSING STRUCTURED DATA (1):"
|
|
21
|
+
assert_includes out.string, "• #{url}"
|
|
22
|
+
refute_includes out.string, "Validation: OK"
|
|
23
|
+
refute JSON.parse(File.read(path))["results"][url]["structured_data_found"]
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
6
27
|
class FakeConfiguration
|
|
7
28
|
attr_accessor :base_url, :concurrency, :fetch_executor, :network_idle_timeout_seconds, :output, :renderer, :timeout_seconds
|
|
8
29
|
|