crawlscope 0.10.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/UPGRADE.md +76 -0
  4. data/docs/architecture.md +73 -0
  5. data/docs/rules.md +49 -0
  6. data/lib/crawlscope/configuration.rb +2 -1
  7. data/lib/crawlscope/crawl.rb +11 -79
  8. data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
  9. data/lib/crawlscope/crawling/target.rb +45 -0
  10. data/lib/crawlscope/crawling/target_resolver.rb +75 -0
  11. data/lib/crawlscope/documents/canonical.rb +22 -0
  12. data/lib/crawlscope/documents/robots.rb +57 -0
  13. data/lib/crawlscope/page.rb +4 -0
  14. data/lib/crawlscope/reporter.rb +6 -160
  15. data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
  16. data/lib/crawlscope/reporting/url_formatter.rb +29 -0
  17. data/lib/crawlscope/rule_registry.rb +2 -2
  18. data/lib/crawlscope/rules/content_quality.rb +18 -0
  19. data/lib/crawlscope/rules/indexability.rb +9 -14
  20. data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
  21. data/lib/crawlscope/rules/links/extractor.rb +100 -0
  22. data/lib/crawlscope/rules/links/graph.rb +39 -0
  23. data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
  24. data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
  25. data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
  26. data/lib/crawlscope/rules/links/scope.rb +23 -0
  27. data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
  28. data/lib/crawlscope/rules/links/url_checks.rb +54 -0
  29. data/lib/crawlscope/rules/links.rb +18 -600
  30. data/lib/crawlscope/rules/metadata.rb +2 -2
  31. data/lib/crawlscope/rules/structured_data.rb +26 -77
  32. data/lib/crawlscope/rules/uniqueness.rb +1 -1
  33. data/lib/crawlscope/schemas.rb +10 -2
  34. data/lib/crawlscope/server_timing/reporter.rb +2 -13
  35. data/lib/crawlscope/structured_data/audit.rb +17 -76
  36. data/lib/crawlscope/structured_data/check.rb +2 -0
  37. data/lib/crawlscope/structured_data/document.rb +14 -1
  38. data/lib/crawlscope/structured_data/entry.rb +31 -0
  39. data/lib/crawlscope/structured_data/outcome.rb +11 -0
  40. data/lib/crawlscope/structured_data/report.rb +6 -14
  41. data/lib/crawlscope/structured_data/reporter.rb +6 -11
  42. data/lib/crawlscope/structured_data/validator.rb +64 -0
  43. data/lib/crawlscope/version.rb +1 -1
  44. data/test/crawlscope/cli_test.rb +21 -0
  45. data/test/crawlscope/content_quality_rule_test.rb +51 -0
  46. data/test/crawlscope/links_rule_test.rb +16 -0
  47. data/test/crawlscope/reporter_test.rb +12 -0
  48. data/test/crawlscope/schema_registry_test.rb +22 -0
  49. data/test/crawlscope/structured_data_audit_test.rb +60 -0
  50. data/test/crawlscope/structured_data_report_test.rb +32 -0
  51. data/test/crawlscope/structured_data_rule_test.rb +11 -1
  52. data/test/crawlscope/structured_data_validator_test.rb +24 -0
  53. metadata +22 -1
@@ -3,11 +3,11 @@
3
3
  module Crawlscope
4
4
  module Rules
5
5
  class StructuredData
6
- CAREER_DETAIL_PATH = %r{/careers/[^/]+/?\z}
7
-
8
6
  attr_reader :code
9
7
 
10
- def initialize
8
+ def initialize(job_posting_path: nil, required_types: nil)
9
+ @job_posting_path = job_posting_path
10
+ @required_types = required_types
11
11
  @code = :structured_data
12
12
  end
13
13
 
@@ -24,8 +24,10 @@ module Crawlscope
24
24
  private
25
25
 
26
26
  def validate_page(page, issues, schema_registry)
27
- document = Crawlscope::StructuredData::Document.new(html: page.body)
28
- items = document.items
27
+ validation = Crawlscope::StructuredData::Validator.new(schema_registry: schema_registry).call(
28
+ html: page.body, required_types: @required_types ? @required_types.call(page.url) : []
29
+ )
30
+ items = validation.items
29
31
 
30
32
  if items.empty?
31
33
  issues.add(
@@ -39,38 +41,26 @@ module Crawlscope
39
41
  return
40
42
  end
41
43
 
42
- items.each do |item|
43
- data = item.data
44
- source = item.source
45
-
46
- if data.is_a?(Hash) && data[:error]
47
- issues.add(
48
- code: :structured_data_parse_error,
49
- severity: :warning,
50
- category: :structured_data,
51
- url: page.url,
52
- message: "#{source_label(source)} parse error: #{data[:message]}",
53
- details: {source: source}
54
- )
55
- next
44
+ validation.findings.each do |finding|
45
+ details = {source: finding.source}
46
+ message = case finding.code
47
+ when :structured_data_parse_error
48
+ "#{source_label(finding.source)} parse error: #{finding.errors.first[:issue]}"
49
+ when :structured_data_missing_type
50
+ details[:paths] = finding.paths
51
+ "#{source_label(finding.source)} structured data missing @type"
52
+ when :structured_data_missing_required_type
53
+ details[:expected_types] = finding.errors.first.fetch(:expected_types)
54
+ "page missing required structured data: #{finding.type}"
55
+ else
56
+ details[:errors] = finding.errors
57
+ schema_error_message(finding.source, finding.errors)
56
58
  end
57
-
58
- validate_type_presence(page, source, data, issues)
59
-
60
- errors = schema_registry.validate(data)
61
- next if errors.empty?
62
-
63
- issues.add(
64
- code: :structured_data_schema_error,
65
- severity: :warning,
66
- category: :structured_data,
67
- url: page.url,
68
- message: schema_error_message(source, errors),
69
- details: {errors: errors, source: source}
70
- )
59
+ issues.add(code: finding.code, severity: :warning, category: :structured_data,
60
+ url: page.url, message: message, details: details)
71
61
  end
72
62
 
73
- validate_job_posting_count(page, items, issues)
63
+ validate_job_posting_count(page, items, issues) if @job_posting_path && career_detail_page?(page.url)
74
64
  end
75
65
 
76
66
  def source_label(source)
@@ -83,7 +73,7 @@ module Crawlscope
83
73
  end
84
74
 
85
75
  def validate_job_posting_count(page, items, issues)
86
- job_postings = items.select { |item| structured_data_types(item.data).include?("JobPosting") }
76
+ job_postings = items.select { |item| item.types.include?("JobPosting") }
87
77
  return if job_postings.size == 1
88
78
 
89
79
  if job_postings.size > 1
@@ -107,49 +97,8 @@ module Crawlscope
107
97
  end
108
98
  end
109
99
 
110
- def validate_type_presence(page, source, data, issues)
111
- missing_paths = missing_type_paths(data)
112
- return if missing_paths.empty?
113
-
114
- issues.add(
115
- code: :structured_data_missing_type,
116
- severity: :warning,
117
- category: :structured_data,
118
- url: page.url,
119
- message: "#{source_label(source)} structured data missing @type",
120
- details: {paths: missing_paths, source: source}
121
- )
122
- end
123
-
124
- def missing_type_paths(data, path = "$")
125
- return [] unless data.is_a?(Hash)
126
-
127
- paths = []
128
- paths << path if data["@type"].to_s.strip.empty?
129
-
130
- if data["@graph"].is_a?(Array)
131
- data["@graph"].each_with_index do |entry, index|
132
- paths.concat(missing_type_paths(entry, "#{path}.@graph[#{index}]"))
133
- end
134
- end
135
-
136
- paths
137
- end
138
-
139
- def structured_data_types(data)
140
- return [] unless data.is_a?(Hash)
141
-
142
- types = Array(data["@type"]).map(&:to_s)
143
-
144
- if data["@graph"].is_a?(Array)
145
- types.concat(data["@graph"].flat_map { |entry| structured_data_types(entry) })
146
- end
147
-
148
- types
149
- end
150
-
151
100
  def career_detail_page?(url)
152
- URI(url).path.match?(CAREER_DETAIL_PATH)
101
+ URI(url).path.match?(@job_posting_path)
153
102
  rescue URI::InvalidURIError
154
103
  false
155
104
  end
@@ -57,7 +57,7 @@ module Crawlscope
57
57
 
58
58
  {
59
59
  content_fingerprint_digest: content_fingerprint_digest(page.doc),
60
- canonical: page.doc.at_css('link[rel="canonical"]')&.[]("href").to_s.strip,
60
+ canonical: page.canonical.href,
61
61
  description: page.doc.at_css('meta[name="description"]')&.[]("content").to_s.strip,
62
62
  shingles: shingles_for(tokens),
63
63
  title: page.doc.at_css("title")&.text.to_s.strip,
@@ -2,6 +2,14 @@
2
2
 
3
3
  module Crawlscope
4
4
  class Schemas
5
+ # Draft 4 omits date; the native Draft 3 format checks real calendar dates.
6
+ DATE_OR_DATE_TIME = {
7
+ anyOf: [
8
+ {type: "string", format: "date-time"},
9
+ {"$schema" => "http://json-schema.org/draft-03/schema#", "type" => "string", "format" => "date"}
10
+ ]
11
+ }.freeze
12
+
5
13
  FAQ_PAGE = {
6
14
  "type" => "object",
7
15
  "required" => ["@context", "@type", "mainEntity"],
@@ -123,7 +131,7 @@ module Crawlscope
123
131
  :itemReviewed => {type: "object"},
124
132
  :reviewRating => RATING,
125
133
  :author => {type: ["object", "string"]},
126
- :datePublished => {type: "string", format: "date-time"},
134
+ :datePublished => DATE_OR_DATE_TIME,
127
135
  :reviewBody => {type: "string"}
128
136
  }
129
137
  }.freeze
@@ -136,7 +144,7 @@ module Crawlscope
136
144
  :reviewRating => RATING,
137
145
  :author => {type: ["object", "string"]},
138
146
  :reviewBody => {type: "string"},
139
- :datePublished => {type: "string", format: "date-time"}
147
+ :datePublished => DATE_OR_DATE_TIME
140
148
  }
141
149
  }.freeze
142
150
 
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "uri"
4
-
5
3
  module Crawlscope
6
4
  class ServerTiming
7
5
  class Reporter
@@ -71,11 +69,12 @@ module Crawlscope
71
69
  return if samples.empty?
72
70
 
73
71
  @io.puts(" Pages above #{SLOW_PAGE_THRESHOLD_MS} ms:")
72
+ urls = Reporting::UrlFormatter.new(base_url: base_url)
74
73
  samples.each do |sample|
75
74
  metric = sample[:metric]
76
75
  detail = metric.description ? " #{description(metric.description)}" : ""
77
76
  @io.puts(
78
- " #{relative_url(sample[:page].url, base_url: base_url)}: " \
77
+ " #{urls.call(sample[:page].url)}: " \
79
78
  "#{duration(metric.duration)} (#{metric.name}#{detail})"
80
79
  )
81
80
  end
@@ -96,16 +95,6 @@ module Crawlscope
96
95
  format("%.1f%%", numerator.fdiv(denominator) * 100)
97
96
  end
98
97
 
99
- def relative_url(url, base_url:)
100
- uri = URI.parse(url)
101
- base_uri = URI.parse(base_url)
102
- return url unless uri.host == base_uri.host && uri.scheme == base_uri.scheme && uri.port == base_uri.port
103
-
104
- uri.request_uri
105
- rescue URI::InvalidURIError
106
- url
107
- end
108
-
109
98
  def pluralize(word, count)
110
99
  (count == 1) ? word.to_s : "#{word}s"
111
100
  end
@@ -3,37 +3,17 @@
3
3
  module Crawlscope
4
4
  module StructuredData
5
5
  class Audit
6
- Page = Data.define(:url, :status, :structured_items, :errors, :fetch_error, :content_type, :skipped_reason) do
7
- def json_ld_count
8
- structured_items.count { |item| item[:source] == "json-ld" }
9
- end
10
-
11
- def microdata_count
12
- structured_items.count { |item| item[:source] == "microdata" }
13
- end
14
-
15
- def ok?
16
- fetch_error.nil? && errors.empty?
17
- end
18
-
19
- def structured_data_found?
20
- skipped_reason == "non-html" || structured_items.any?
21
- end
22
- end
23
-
24
- Outcome = Data.define(:entries) do
25
- def ok?
26
- entries.all?(&:ok?)
27
- end
28
- end
6
+ Page = Entry
7
+ Outcome = Crawlscope::StructuredData::Outcome
29
8
 
30
- def initialize(schema_registry:, renderer:, timeout_seconds:, browser_factory: nil, network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS, scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE)
9
+ def initialize(schema_registry:, renderer:, timeout_seconds:, browser_factory: nil, network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS, scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE, required_types: nil)
31
10
  @schema_registry = schema_registry
32
11
  @renderer = renderer.to_sym
33
12
  @timeout_seconds = timeout_seconds
34
13
  @browser_factory = browser_factory
35
14
  @network_idle_timeout_seconds = network_idle_timeout_seconds
36
15
  @scroll_page = scroll_page
16
+ @required_types = required_types
37
17
  end
38
18
 
39
19
  def call(urls:)
@@ -47,68 +27,29 @@ module Crawlscope
47
27
 
48
28
  private
49
29
 
50
- def build_browser(base_url)
51
- browser_factory = @browser_factory
52
-
53
- if browser_factory
54
- browser_factory.call
55
- else
56
- Crawlscope::Browser.new(
57
- base_url: base_url,
58
- timeout_seconds: @timeout_seconds,
59
- network_idle_timeout_seconds: @network_idle_timeout_seconds,
60
- scroll_page: @scroll_page
61
- )
62
- end
63
- rescue LoadError => error
64
- raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
65
- end
66
-
67
30
  def build_fetcher(urls)
68
- first_url = urls.first.to_s
69
- base_url = first_url.empty? ? "http://localhost:3000" : first_url
70
-
71
- if @renderer == :browser
72
- build_browser(base_url)
73
- else
74
- Crawlscope::Http.new(base_url: base_url, timeout_seconds: @timeout_seconds)
75
- end
31
+ Crawling::FetcherFactory.new(
32
+ base_url: urls.first || "http://localhost:3000", renderer: @renderer,
33
+ timeout_seconds: @timeout_seconds, browser_factory: @browser_factory,
34
+ network_idle_timeout_seconds: @network_idle_timeout_seconds,
35
+ scroll_page: @scroll_page
36
+ ).call
76
37
  end
77
38
 
78
39
  def build_validation_errors(page)
79
- document = Document.new(html: page.body)
80
- structured_items = document.items.map do |item|
81
- {
82
- data: item.data,
83
- source: item.source
84
- }
85
- end
86
-
87
- errors = structured_items.filter_map do |item|
88
- if item[:data].is_a?(Hash) && item[:data][:error]
89
- {
90
- errors: [{field: "parse", issue: item[:data][:message]}],
91
- source: item[:source],
92
- type: item[:source]
93
- }
94
- else
95
- schema_errors = @schema_registry.validate(item[:data])
96
- next if schema_errors.empty?
97
-
98
- {
99
- errors: schema_errors,
100
- source: item[:source],
101
- type: item[:data]["@type"] || item[:source]
102
- }
103
- end
40
+ validation = Validator.new(schema_registry: @schema_registry).call(
41
+ html: page.body, required_types: @required_types ? @required_types.call(page.url) : []
42
+ )
43
+ structured_items = validation.items.map { |item| {data: item.data, source: item.source} }
44
+ errors = validation.findings.map do |finding|
45
+ {errors: finding.errors, source: finding.source, type: finding.type}
104
46
  end
105
-
106
47
  [structured_items, errors]
107
48
  end
108
49
 
109
50
  def validate_url(url, fetcher)
110
51
  page = fetcher.fetch(url)
111
- content_type = page.headers["content-type"].to_s
52
+ content_type = page.header("content-type").to_s
112
53
 
113
54
  if page.error
114
55
  Page.new(url: url, status: page.status, structured_items: [], errors: [], fetch_error: page.error, content_type: content_type, skipped_reason: nil)
@@ -5,6 +5,7 @@ module Crawlscope
5
5
  class Check
6
6
  def initialize(configuration:)
7
7
  @configuration = configuration
8
+ @required_types = configuration.required_structured_data_types if configuration.respond_to?(:required_structured_data_types)
8
9
  end
9
10
 
10
11
  def call(urls:, debug: false, renderer: @configuration.renderer, timeout_seconds: @configuration.timeout_seconds, report_path: nil, summary: false)
@@ -25,6 +26,7 @@ module Crawlscope
25
26
  browser_factory: @configuration.browser_factory,
26
27
  network_idle_timeout_seconds: @configuration.network_idle_timeout_seconds,
27
28
  renderer: renderer,
29
+ required_types: @required_types,
28
30
  schema_registry: @configuration.schema_registry,
29
31
  scroll_page: @configuration.scroll_page?,
30
32
  timeout_seconds: timeout_seconds
@@ -6,7 +6,20 @@ require "nokogiri"
6
6
  module Crawlscope
7
7
  module StructuredData
8
8
  class Document
9
- Item = Data.define(:source, :data)
9
+ Item = Data.define(:source, :data) do
10
+ def types
11
+ nodes = [data]
12
+ types = []
13
+ until nodes.empty?
14
+ node = nodes.pop
15
+ next unless node.is_a?(Hash)
16
+
17
+ types.concat(Array(node["@type"]))
18
+ nodes.concat(Array(node["@graph"]))
19
+ end
20
+ types
21
+ end
22
+ end
10
23
 
11
24
  def initialize(html:)
12
25
  @doc = Nokogiri::HTML(html.to_s)
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module StructuredData
5
+ Entry = Data.define(:url, :status, :structured_items, :errors, :fetch_error, :content_type, :skipped_reason) do
6
+ def json_ld_count
7
+ structured_items.count { |item| item[:source] == "json-ld" }
8
+ end
9
+
10
+ def microdata_count
11
+ structured_items.count { |item| item[:source] == "microdata" }
12
+ end
13
+
14
+ def ok?
15
+ failure_kind.nil?
16
+ end
17
+
18
+ def failure_kind
19
+ return :http if fetch_error
20
+ return :missing_data unless structured_data_found?
21
+ return :validation if errors.any?
22
+
23
+ nil
24
+ end
25
+
26
+ def structured_data_found?
27
+ skipped_reason == "non-html" || structured_items.any?
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module StructuredData
5
+ Outcome = Data.define(:entries) do
6
+ def ok?
7
+ entries.all?(&:ok?)
8
+ end
9
+ end
10
+ end
11
+ end
@@ -8,19 +8,19 @@ module Crawlscope
8
8
  end
9
9
 
10
10
  def all_valid?
11
- http_errors.empty? && missing_data.empty? && validation_errors.empty?
11
+ @result.ok?
12
12
  end
13
13
 
14
14
  def failure_count
15
- http_errors.size + missing_data.size + validation_errors.size
15
+ entries.count { |entry| !entry.ok? }
16
16
  end
17
17
 
18
18
  def http_errors
19
- entries.select { |entry| entry.fetch_error && entry.status != 200 }
19
+ entries.select { |entry| entry.failure_kind == :http }
20
20
  end
21
21
 
22
22
  def missing_data
23
- entries.select { |entry| entry.status == 200 && !entry.structured_data_found? }
23
+ entries.select { |entry| entry.failure_kind == :missing_data }
24
24
  end
25
25
 
26
26
  def results
@@ -34,7 +34,7 @@ module Crawlscope
34
34
  end
35
35
 
36
36
  def validation_errors
37
- entries.select { |entry| entry.status == 200 && entry.errors.any? }
37
+ entries.select { |entry| entry.failure_kind == :validation }
38
38
  end
39
39
 
40
40
  private
@@ -44,15 +44,7 @@ module Crawlscope
44
44
  end
45
45
 
46
46
  def result_for(entry)
47
- if entry.fetch_error && entry.status == 200
48
- {
49
- status: entry.status,
50
- error: entry.fetch_error,
51
- structured_data_found: false,
52
- validation_errors: [],
53
- json_ld_count: 0
54
- }
55
- elsif entry.fetch_error
47
+ if entry.fetch_error
56
48
  {
57
49
  status: entry.status || "exception",
58
50
  error: entry.fetch_error,
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "uri"
4
3
  require "json"
5
4
 
6
5
  module Crawlscope
@@ -36,12 +35,6 @@ module Crawlscope
36
35
 
37
36
  private
38
37
 
39
- def extract_path(url)
40
- URI.parse(url).path
41
- rescue URI::InvalidURIError
42
- url
43
- end
44
-
45
38
  def print_category(name, items)
46
39
  return if items.empty?
47
40
 
@@ -56,15 +49,15 @@ module Crawlscope
56
49
  @io.puts("")
57
50
 
58
51
  print_category("HTTP ERRORS", report.http_errors) do |entry|
59
- @io.puts("• #{extract_path(entry.url)} (#{entry.status}: #{entry.fetch_error})")
52
+ @io.puts("• #{entry.url} (#{entry.status}: #{entry.fetch_error})")
60
53
  end
61
54
 
62
55
  print_category("MISSING STRUCTURED DATA", report.missing_data) do |entry|
63
- @io.puts("• #{extract_path(entry.url)}")
56
+ @io.puts("• #{entry.url}")
64
57
  end
65
58
 
66
59
  print_category("VALIDATION ERRORS", report.validation_errors) do |entry|
67
- @io.puts("• #{extract_path(entry.url)}")
60
+ @io.puts("• #{entry.url}")
68
61
 
69
62
  entry.errors.each do |error|
70
63
  @io.puts(" #{source_label(error[:source])}:")
@@ -121,7 +114,9 @@ module Crawlscope
121
114
 
122
115
  def report_validation(entry)
123
116
  @io.puts("")
124
- if entry.errors.empty?
117
+ if !entry.structured_data_found?
118
+ @io.puts("Validation: FAILED (no structured data found)")
119
+ elsif entry.errors.empty?
125
120
  @io.puts("Validation: OK")
126
121
  else
127
122
  @io.puts("Validation: FAILED")
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module StructuredData
5
+ class Validator
6
+ Finding = Data.define(:code, :source, :type, :errors, :paths)
7
+ Validation = Data.define(:items, :findings)
8
+
9
+ def initialize(schema_registry:)
10
+ @schema_registry = schema_registry
11
+ end
12
+
13
+ def call(html:, required_types: [])
14
+ items = Document.new(html: html).items
15
+ findings = items.flat_map { |item| validate(item) }
16
+ types = items.flat_map(&:types)
17
+ Array(required_types).each do |requirement|
18
+ alternatives = Array(requirement)
19
+ next if (alternatives & types).any?
20
+
21
+ expected = alternatives.join(" or ")
22
+ findings << Finding.new(code: :structured_data_missing_required_type, source: "page", type: expected,
23
+ errors: [{field: "@type", issue: "must include #{expected}", expected_types: alternatives}], paths: [])
24
+ end
25
+ Validation.new(items: items, findings: findings)
26
+ end
27
+
28
+ private
29
+
30
+ def validate(item)
31
+ data = item.data
32
+ if data.is_a?(Hash) && data[:error]
33
+ return [finding(:structured_data_parse_error, item, [{field: "parse", issue: data[:message]}])]
34
+ end
35
+
36
+ findings = []
37
+ paths = missing_type_paths(data)
38
+ unless paths.empty?
39
+ errors = paths.map { |path| {field: "#{path}.@type", issue: "is required"} }
40
+ findings << finding(:structured_data_missing_type, item, errors, paths)
41
+ end
42
+ errors = @schema_registry.validate(data)
43
+ findings << finding(:structured_data_schema_error, item, errors) unless errors.empty?
44
+ findings
45
+ end
46
+
47
+ def finding(code, item, errors, paths = [])
48
+ Finding.new(code: code, source: item.source, type: item.data["@type"] || item.source, errors: errors, paths: paths)
49
+ end
50
+
51
+ def missing_type_paths(data, path = "$")
52
+ return [] unless data.is_a?(Hash)
53
+
54
+ paths = data["@type"].to_s.strip.empty? ? [path] : []
55
+ if data["@graph"].is_a?(Array)
56
+ data["@graph"].each_with_index do |entry, index|
57
+ paths.concat(missing_type_paths(entry, "#{path}.@graph[#{index}]"))
58
+ end
59
+ end
60
+ paths
61
+ end
62
+ end
63
+ end
64
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Crawlscope
4
- VERSION = "0.10.1"
4
+ VERSION = "0.10.2"
5
5
  end
@@ -3,6 +3,27 @@
3
3
  require "test_helper"
4
4
 
5
5
  class CrawlscopeCliTest < Minitest::Test
6
+ def test_ldjson_missing_data_fails_in_details_summary_json_and_exit_status
7
+ url = "https://example.com/page?locale=en"
8
+ stub_request(:get, url).to_return(status: 200, headers: {"Content-Type" => "text/html"}, body: "<h1>No markup</h1>")
9
+ out = StringIO.new
10
+ err = StringIO.new
11
+
12
+ Dir.mktmpdir do |directory|
13
+ path = File.join(directory, "report.json")
14
+ status = Crawlscope::Cli.start(["ldjson", "--url", url, "--summary", "--report-path", path], out: out, err: err)
15
+
16
+ assert_equal 1, status
17
+ assert_empty err.string
18
+ assert_includes out.string, "Validation: FAILED (no structured data found)"
19
+ assert_includes out.string, "STATUS: FAILED"
20
+ assert_includes out.string, "MISSING STRUCTURED DATA (1):"
21
+ assert_includes out.string, "• #{url}"
22
+ refute_includes out.string, "Validation: OK"
23
+ refute JSON.parse(File.read(path))["results"][url]["structured_data_found"]
24
+ end
25
+ end
26
+
6
27
  class FakeConfiguration
7
28
  attr_accessor :base_url, :concurrency, :fetch_executor, :network_idle_timeout_seconds, :output, :renderer, :timeout_seconds
8
29