crawlscope 0.10.1 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +18 -0
  3. data/UPGRADE.md +76 -0
  4. data/docs/architecture.md +73 -0
  5. data/docs/rules.md +49 -0
  6. data/lib/crawlscope/configuration.rb +2 -1
  7. data/lib/crawlscope/crawl.rb +11 -79
  8. data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
  9. data/lib/crawlscope/crawling/target.rb +45 -0
  10. data/lib/crawlscope/crawling/target_resolver.rb +75 -0
  11. data/lib/crawlscope/documents/canonical.rb +22 -0
  12. data/lib/crawlscope/documents/robots.rb +57 -0
  13. data/lib/crawlscope/page.rb +4 -0
  14. data/lib/crawlscope/reporter.rb +6 -160
  15. data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
  16. data/lib/crawlscope/reporting/url_formatter.rb +29 -0
  17. data/lib/crawlscope/rule_registry.rb +2 -2
  18. data/lib/crawlscope/rules/content_quality.rb +18 -0
  19. data/lib/crawlscope/rules/indexability.rb +9 -14
  20. data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
  21. data/lib/crawlscope/rules/links/extractor.rb +100 -0
  22. data/lib/crawlscope/rules/links/graph.rb +39 -0
  23. data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
  24. data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
  25. data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
  26. data/lib/crawlscope/rules/links/scope.rb +23 -0
  27. data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
  28. data/lib/crawlscope/rules/links/url_checks.rb +54 -0
  29. data/lib/crawlscope/rules/links.rb +18 -600
  30. data/lib/crawlscope/rules/metadata.rb +2 -2
  31. data/lib/crawlscope/rules/structured_data.rb +26 -77
  32. data/lib/crawlscope/rules/uniqueness.rb +1 -1
  33. data/lib/crawlscope/schemas.rb +10 -2
  34. data/lib/crawlscope/server_timing/reporter.rb +2 -13
  35. data/lib/crawlscope/structured_data/audit.rb +17 -76
  36. data/lib/crawlscope/structured_data/check.rb +2 -0
  37. data/lib/crawlscope/structured_data/document.rb +14 -1
  38. data/lib/crawlscope/structured_data/entry.rb +31 -0
  39. data/lib/crawlscope/structured_data/outcome.rb +11 -0
  40. data/lib/crawlscope/structured_data/report.rb +6 -14
  41. data/lib/crawlscope/structured_data/reporter.rb +6 -11
  42. data/lib/crawlscope/structured_data/validator.rb +64 -0
  43. data/lib/crawlscope/version.rb +1 -1
  44. data/test/crawlscope/cli_test.rb +21 -0
  45. data/test/crawlscope/content_quality_rule_test.rb +51 -0
  46. data/test/crawlscope/links_rule_test.rb +16 -0
  47. data/test/crawlscope/reporter_test.rb +12 -0
  48. data/test/crawlscope/schema_registry_test.rb +22 -0
  49. data/test/crawlscope/structured_data_audit_test.rb +60 -0
  50. data/test/crawlscope/structured_data_report_test.rb +32 -0
  51. data/test/crawlscope/structured_data_rule_test.rb +11 -1
  52. data/test/crawlscope/structured_data_validator_test.rb +24 -0
  53. metadata +22 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1ff240a395648f3047e4fd24fe2728ad5c7ad3c22406029f7dbb0316f43bee73
4
- data.tar.gz: 93e012e28037929afadba5df00f8ad441665ddda8cbc55775278fcc427d5af00
3
+ metadata.gz: 9f1e7d817608a6023e7230e5568408576f375733ff0e466ab0ae9119b72cae8a
4
+ data.tar.gz: 4c645137855354cdb4847e641d78969a2f6014698d142c7a446b5555d4cbfbbb
5
5
  SHA512:
6
- metadata.gz: d7f51921917c37fd1cb6576d1c1c56957cdc6be141906b5958153c548296248a76316832efd7dcd562835a08b6393a3009e577ed59f676b72786ec3dc99cd81b
7
- data.tar.gz: 7db8cd080f1256e42e17cf09cbd3a34f2a62bcbd0e48238e58f420359519fd4e737516c21ad48212c17bc34e093914e641d2a8fbbd60ec94f68600466ae098bf
6
+ metadata.gz: e341bfed169b466693ec2bf72aec7ca53bc4c3ebabbc5935491ecd523b2edfb594a1bc44e16b3c22499c4a97ac4a1b70c4f0f42012933f652e7ea4bd1248c301
7
+ data.tar.gz: 6421dbc9dc6e15882b08a61f1a73ee23f2cdd02dbfdfc87136be363d65ac844d852065b6ef1041908278008627ecc740f66e979fe9be7ec57cca6fa71f158b90
data/CHANGELOG.md CHANGED
@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
+ ## [0.10.2] - 2026-11-15
9
+
10
+
11
+ ### Added
12
+
13
+ - unify validation and reporting responsibilities
14
+
15
+ - enforce required structured data types by URL
16
+
17
+
18
+
19
+
20
+ ### Fixed
21
+
22
+ - accept date-only review publication dates
23
+
24
+
25
+
8
26
  ## [0.10.1] - 2026-11-08
9
27
 
10
28
 
data/UPGRADE.md CHANGED
@@ -37,6 +37,82 @@ in the README; the shared workflow does not inherit another gem's permission.
37
37
 
38
38
  ## Next Release
39
39
 
40
+ ### Review publication dates
41
+
42
+ Standalone and nested reviews now accept a date (`2026-09-16`) or timestamp for
43
+ `datePublished`, matching [Schema.org's Date or DateTime contract](https://schema.org/datePublished).
44
+ Invalid calendar dates still fail validation. Hosts no longer need to invent a
45
+ time to validate a review whose source publishes only its date.
46
+
47
+ ### Required structured-data types
48
+
49
+ Set `config.required_structured_data_types` to a callback accepting a URL and
50
+ returning the types that page must contain. Separate strings require all types;
51
+ nested arrays accept alternatives, such as `[["Article", "BlogPosting",
52
+ "NewsArticle"]]`. The default remains no additional type requirements.
53
+ See [the rule configuration](docs/rules.md#required-types-by-url) for examples.
54
+
55
+ The default sitemap rule and direct URL audits apply the same requirements.
56
+ Custom rule registries must pass the callback through `required_types:` when
57
+ constructing their structured-data rule. Custom configuration objects supplied
58
+ to `Run` or `StructuredData::Check` can expose `required_structured_data_types`
59
+ to opt in. Existing objects without that optional setting retain their behavior.
60
+
61
+ ### Shared audit outcomes and internal architecture
62
+
63
+ Direct structured-data audits now fail when an HTML page has no structured data
64
+ or contains a JSON-LD or microdata item missing `@type`. Detailed output,
65
+ summaries, and CLI/Rake exit status agree on these failures. Summary categories
66
+ also include failures on successful responses other than HTTP 200 and browser
67
+ responses without a status. Non-HTML responses remain successful skips.
68
+
69
+ Hosts using `validate_json_ld` or `crawlscope ldjson` must add the missing markup
70
+ or type to pass validation. Summary URL rows now retain the full URL, including
71
+ the host and query string; update text parsers that expected path-only rows.
72
+ The JSON report retains its existing shape and includes missing-type errors in
73
+ `validation_errors`.
74
+
75
+ JobPosting count checks are now opt-in and scoped to a host-supplied path
76
+ pattern. Crawlscope no longer assumes `/careers/...` identifies job detail
77
+ pages or that multiple JobPosting items are invalid on other pages. Hosts that
78
+ need exactly one JobPosting on job detail pages can replace the default rule:
79
+
80
+ ```ruby
81
+ Crawlscope.configure do |config|
82
+ config.rule_registry = -> {
83
+ registry = Crawlscope::RuleRegistry.default(site_name: config.site_name)
84
+ rules = registry.rules.map do |rule|
85
+ if rule.code == :structured_data
86
+ Crawlscope::Rules::StructuredData.new(job_posting_path: %r{/careers/[^/]+/?\z})
87
+ else
88
+ rule
89
+ end
90
+ end
91
+ Crawlscope::RuleRegistry.new(rules: rules, default_codes: registry.default_codes)
92
+ }
93
+ end
94
+ ```
95
+
96
+ Use your application's actual job path pattern. The former
97
+ `Rules::StructuredData::CAREER_DETAIL_PATH` constant is removed; pass the pattern
98
+ directly when constructing the rule.
99
+
100
+ Existing `Crawl`, `Reporter`, rule entry points, issue fields, and
101
+ `StructuredData::Audit::Page` / `Outcome` constructors remain available.
102
+ Internal responsibilities now live under `Crawling`, `Documents`, `Reporting`,
103
+ and `Rules::Links`; see [architecture](docs/architecture.md) for extension points.
104
+
105
+ ### Images missing alt attributes
106
+
107
+ The default `content_quality` rule now emits one `missing_image_alt` warning per
108
+ HTML page containing `<img>` elements without an `alt` attribute. Empty
109
+ `alt=""` attributes remain valid for decorative images. Details include `count`
110
+ and `image_sources`; text reports list the affected image sources.
111
+
112
+ No host configuration changes are required. Applications that count warnings or
113
+ fail crawls on warnings should expect this additional check. Add appropriate alt
114
+ text to informative images or `alt=""` to decorative images to resolve warnings.
115
+
40
116
  ### Query links retain their own URL identity
41
117
 
42
118
  URL normalization now preserves query strings and still removes fragments.
@@ -0,0 +1,73 @@
1
+ # Architecture
2
+
3
+ Crawlscope separates fetching, document interpretation, audit policy, and
4
+ presentation. Public entry points remain small coordinators. The directories
5
+ below contain the collaborators they use:
6
+
7
+ ```text
8
+ lib/crawlscope/
9
+ crawling/
10
+ fetcher_factory.rb HTTP/browser construction and browser load errors
11
+ target_resolver.rb URL normalization, page reuse, and target caching
12
+ target.rb Resolved target status and indexability
13
+ documents/
14
+ canonical.rb Canonical href extraction and URL resolution
15
+ robots.rb Shared robots directive interpretation
16
+ reporting/
17
+ issue_presenter.rb Issue headings, rows, evidence, and measurements
18
+ url_formatter.rb URL display that preserves origin and query identity
19
+ rules/
20
+ links.rb Public links rule; assembles one invocation
21
+ links/
22
+ extractor.rb Internal anchor extraction
23
+ scope.rb Shared link inclusion policy
24
+ target_lookup.rb Per-invocation target lookup and batch resolution
25
+ resolution_check.rb Broken and redirected link checks
26
+ graph.rb Inbound counts and source URLs
27
+ inbound_checks.rb Incoming-link policies
28
+ outgoing_checks.rb Outgoing-link and discovered sitemap policies
29
+ canonical_checks.rb Canonical target status and inbound-link policies
30
+ url_checks.rb URL hygiene
31
+ structured_data/
32
+ document.rb JSON-LD and microdata extraction
33
+ validator.rb Shared parse, type-presence, and schema validation
34
+ entry.rb Per-URL outcome and failure classification
35
+ outcome.rb Overall success derived from entries
36
+ audit.rb Fetches direct URLs and adapts validation findings
37
+ report.rb Summary selection and JSON serialization
38
+ ```
39
+
40
+ `Configuration` selects rules and runtime settings. `Run` invokes an audit and
41
+ its reporter. `Crawl` coordinates sitemap expansion, fetching, response checks,
42
+ and rule execution; `Crawling::TargetResolver` supplies rules with cached target
43
+ resolution through the existing `Context` protocol.
44
+
45
+ The links rule creates its collaborators for each call. A reused rule instance
46
+ does not retain a previous crawl's resolver or target cache. `Graph` indexes
47
+ resolved incoming links once, excludes self-links and noindex targets, and
48
+ retains full URL identity. Inbound and canonical checks ask the same graph for
49
+ counts instead of maintaining separate counting algorithms. Path inclusion
50
+ policy lives in `Scope`.
51
+
52
+ Both structured-data entry points use `StructuredData::Validator`. It returns
53
+ items and findings without writing output or choosing command exit status.
54
+ `Rules::StructuredData` translates findings into the sitemap audit's existing
55
+ warning issues. `StructuredData::Audit` translates them into direct-URL entries.
56
+ Host path requirements belong in explicit rule configuration.
57
+
58
+ `StructuredData::Entry` owns failure classification: fetch failure, missing
59
+ markup, validation failure, or success. `Outcome#ok?` and summary categories use
60
+ that classification. The legacy `Audit::Page` and `Audit::Outcome` constants are
61
+ aliases, preserving their constructors and fields.
62
+
63
+ `Reporter` owns text layout and ordering; `Reporting::IssuePresenter` formats
64
+ one issue without performing I/O. `Reporting::UrlFormatter` is shared by crawl
65
+ and timing reports. Structured-data errors use the same `ErrorFormatter` in
66
+ both audit reports. Existing `Issue#message` values remain available to API
67
+ consumers; presenters use structured evidence for richer text rows.
68
+
69
+ When adding a check, put document interpretation in a shared document object,
70
+ policy in a rule, and text formatting in a presenter. Keep network access behind
71
+ the fetcher and resolver interfaces. Add behavior tests at the public rule or
72
+ command boundary; use direct collaborator tests for policies that need their
73
+ own examples. Run `bundle exec rake verify` before handing off changes.
data/docs/rules.md CHANGED
@@ -40,6 +40,53 @@ Checks:
40
40
  - schema validation failures from the configured registry
41
41
  - direct URL structured-data audits through `crawlscope:validate:ldjson`
42
42
 
43
+ Sitemap and direct URL audits share the same document validator, including
44
+ missing `@type` checks. Direct URL audits treat missing markup as a failure;
45
+ sitemap audits retain their warning severity for structured-data findings.
46
+
47
+ ### Required types by URL
48
+
49
+ Schema validation checks the markup a page contains. To detect a missing type
50
+ even when other valid markup exists, configure a URL policy:
51
+
52
+ ```ruby
53
+ Crawlscope.configure do |config|
54
+ config.required_structured_data_types = ->(url) {
55
+ case URI(url).path
56
+ when %r{\A/guides/[^/]+/?\z}
57
+ [["Article", "BlogPosting", "NewsArticle"]]
58
+ when "/guides"
59
+ ["CollectionPage", "ItemList"]
60
+ when %r{/benchmark/?\z}
61
+ ["WebPage", "Dataset"]
62
+ else
63
+ []
64
+ end
65
+ }
66
+ end
67
+ ```
68
+
69
+ Each string requires that type. A nested array accepts any of its types.
70
+ Return an empty array for URLs without a type requirement. The callback receives
71
+ the full URL; the host owns the route policy and content classification.
72
+
73
+ The default sitemap rule and direct URL audits use this policy. A missing
74
+ requirement produces `structured_data_missing_required_type` in sitemap
75
+ findings and a validation failure in direct audits, including their reports and
76
+ exit status. Top-level items, JSON-LD type arrays and `@graph` nodes count;
77
+ an article linked inside an ItemList does not satisfy an article-page requirement.
78
+ Microdata items also count. Custom rule registries must pass the callback to
79
+ `Rules::StructuredData.new(required_types: callback)` themselves.
80
+
81
+ This checks type presence; the schema registry still validates supported types.
82
+ An index, product page or dataset should not acquire Article markup solely
83
+ because an external crawler classified its prose as an article.
84
+
85
+ Job detail pages can opt into an exactly-one-JobPosting policy with
86
+ `Rules::StructuredData.new(job_posting_path: your_path_pattern)` in a custom
87
+ rule registry. No host route patterns are assumed by default. See
88
+ [UPGRADE](../UPGRADE.md) for an example registry configuration.
89
+
43
90
  ## Uniqueness
44
91
 
45
92
  Checks:
@@ -57,6 +104,8 @@ For larger crawls, exact duplicate checks still run and Crawlscope reports
57
104
 
58
105
  Checks:
59
106
 
107
+ - missing `alt` attributes on `<img>` elements throughout the page, including
108
+ headers and footers (empty `alt=""` is allowed for decorative images)
60
109
  - thin visible text
61
110
  - low visible-text-to-HTML ratio
62
111
  - low unique-token ratio
@@ -12,6 +12,7 @@ module Crawlscope
12
12
  DEFAULT_TIMEOUT_SECONDS = 20
13
13
 
14
14
  attr_writer :allowed_statuses, :base_url, :browser_factory, :concurrency, :fetch_executor, :network_idle_timeout_seconds, :output, :profile_token, :renderer, :rule_registry, :schema_registry, :scroll_page, :site_name, :sitemap_path, :timeout_seconds
15
+ attr_accessor :required_structured_data_types
15
16
 
16
17
  def allowed_statuses
17
18
  value = resolve(@allowed_statuses)
@@ -78,7 +79,7 @@ module Crawlscope
78
79
  value = resolve(@rule_registry)
79
80
  return value unless value.nil?
80
81
 
81
- RuleRegistry.default(site_name: site_name)
82
+ RuleRegistry.default(site_name: site_name, required_types: required_structured_data_types)
82
83
  end
83
84
 
84
85
  def audit(base_url: self.base_url, sitemap_path: self.sitemap_path, rule_names: nil)
@@ -28,7 +28,10 @@ module Crawlscope
28
28
  issues = IssueCollection.new
29
29
 
30
30
  collect(pages, issues)
31
- cache(pages)
31
+ @target_resolver = Crawling::TargetResolver.new(
32
+ base_url: @base_url, pages: pages,
33
+ crawler: Crawler.new(page_fetcher: @page_fetcher, concurrency: @concurrency, fetch_executor: @fetch_executor)
34
+ )
32
35
  scan(urls, pages, issues)
33
36
 
34
37
  Result.new(
@@ -58,24 +61,13 @@ module Crawlscope
58
61
  urls
59
62
  end
60
63
 
61
- def browser
62
- Browser.new(
63
- base_url: @base_url,
64
- timeout_seconds: @timeout_seconds,
65
- network_idle_timeout_seconds: @network_idle_timeout_seconds,
66
- profile_token: @profile_token,
67
- scroll_page: @scroll_page
68
- )
69
- rescue LoadError => error
70
- raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
71
- end
72
-
73
64
  def page
74
- if @renderer == :browser
75
- (@browser_factory || method(:browser)).call
76
- else
77
- Http.new(base_url: @base_url, timeout_seconds: @timeout_seconds, adapter: http_adapter, profile_token: @profile_token)
78
- end
65
+ Crawling::FetcherFactory.new(
66
+ base_url: @base_url, renderer: @renderer, timeout_seconds: @timeout_seconds,
67
+ browser_factory: @browser_factory, profile_token: @profile_token,
68
+ network_idle_timeout_seconds: @network_idle_timeout_seconds,
69
+ scroll_page: @scroll_page, adapter: http_adapter
70
+ ).call
79
71
  end
80
72
 
81
73
  def http_adapter
@@ -119,20 +111,6 @@ module Crawlscope
119
111
  end
120
112
  end
121
113
 
122
- def cache(pages)
123
- @pages = {}
124
- @targets = {}
125
-
126
- pages.each do |page|
127
- cache_page(page)
128
- end
129
- end
130
-
131
- def cache_page(page)
132
- @pages[page.normalized_url] = page unless page.normalized_url.to_s.empty?
133
- @pages[page.normalized_final_url] = page unless page.normalized_final_url.to_s.empty?
134
- end
135
-
136
114
  def scan(urls, pages, issues)
137
115
  @rules.each do |rule|
138
116
  rule.call(urls: urls, pages: pages, issues: issues, context: context)
@@ -144,53 +122,7 @@ module Crawlscope
144
122
  end
145
123
 
146
124
  def resolve_all(target_urls)
147
- normalized_by_url = Array(target_urls).to_h do |target_url|
148
- [target_url, Url.normalize(target_url, base_url: @base_url)]
149
- end
150
- normalized_urls = normalized_by_url.values.compact.uniq
151
- missing_urls = []
152
-
153
- normalized_urls.each do |normalized_url|
154
- next if @targets.key?(normalized_url)
155
-
156
- resolved = resolved_page(normalized_url)
157
- if resolved
158
- @targets[normalized_url] = resolved
159
- else
160
- missing_urls << normalized_url
161
- end
162
- end
163
-
164
- fetched_pages(missing_urls).each do |page|
165
- normalized_url = Url.normalize(page.url, base_url: @base_url)
166
- cache_page(page)
167
- @targets[normalized_url] = resolution(page, normalized_url, crawled: false)
168
- end
169
-
170
- normalized_by_url.to_h { |target_url, normalized_url| [target_url, @targets[normalized_url]] }
171
- end
172
-
173
- def fetched_pages(normalized_urls)
174
- return [] if normalized_urls.empty?
175
-
176
- Crawler.new(page_fetcher: @page_fetcher, concurrency: @concurrency, fetch_executor: @fetch_executor).call(normalized_urls)
177
- end
178
-
179
- def resolved_page(normalized_url)
180
- page = @pages[normalized_url]
181
- resolution(page, normalized_url, crawled: true) if page
182
- end
183
-
184
- def resolution(page, normalized_url, crawled:)
185
- {
186
- crawled: crawled,
187
- doc: page.doc,
188
- error: page.error,
189
- final_url: page.normalized_final_url || normalized_url,
190
- headers: page.headers,
191
- html: page.html?,
192
- status: page.status
193
- }
125
+ @target_resolver.call(target_urls)
194
126
  end
195
127
 
196
128
  def redirected?(page)
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Crawling
5
+ class FetcherFactory
6
+ def initialize(base_url:, renderer:, timeout_seconds:, browser_factory: nil,
7
+ network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS,
8
+ scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE, profile_token: nil, adapter: nil)
9
+ @base_url = base_url
10
+ @renderer = renderer
11
+ @timeout_seconds = timeout_seconds
12
+ @browser_factory = browser_factory
13
+ @network_idle_timeout_seconds = network_idle_timeout_seconds
14
+ @scroll_page = scroll_page
15
+ @profile_token = profile_token
16
+ @adapter = adapter
17
+ end
18
+
19
+ def call
20
+ return Http.new(base_url: @base_url, timeout_seconds: @timeout_seconds, adapter: @adapter, profile_token: @profile_token) unless @renderer == :browser
21
+ return @browser_factory.call if @browser_factory
22
+
23
+ Browser.new(base_url: @base_url, timeout_seconds: @timeout_seconds,
24
+ network_idle_timeout_seconds: @network_idle_timeout_seconds,
25
+ scroll_page: @scroll_page, profile_token: @profile_token)
26
+ rescue LoadError => error
27
+ raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Crawling
5
+ Target = Data.define(:target_url, :resolution) do
6
+ def allowed?(statuses)
7
+ statuses.include?(status)
8
+ end
9
+
10
+ def final_path
11
+ Url.path(final_url)
12
+ end
13
+
14
+ def final_url
15
+ value = resolution[:final_url].to_s
16
+ value.empty? ? target_url : value
17
+ end
18
+
19
+ def ignored_error?
20
+ resolution && status.nil? && resolution[:crawled] && resolution[:error]
21
+ end
22
+
23
+ def html?
24
+ resolution && resolution[:html]
25
+ end
26
+
27
+ def noindex?
28
+ Documents::Robots.noindex_header?(resolution[:headers] || {}) ||
29
+ Documents::Robots.noindex_meta?(resolution[:doc])
30
+ end
31
+
32
+ def status
33
+ resolution && resolution[:status]
34
+ end
35
+
36
+ def redirect?
37
+ (status && (300..399).cover?(status.to_i)) || final_url != target_url
38
+ end
39
+
40
+ def unresolved?
41
+ resolution.nil? || (status.nil? && !ignored_error?)
42
+ end
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Crawling
5
+ class TargetResolver
6
+ def initialize(base_url:, crawler:, pages:)
7
+ @base_url = base_url
8
+ @crawler = crawler
9
+ @pages = {}
10
+ @targets = {}
11
+
12
+ pages.each do |page|
13
+ cache_page(page)
14
+ end
15
+ end
16
+
17
+ def call(target_urls)
18
+ normalized_by_url = Array(target_urls).to_h do |target_url|
19
+ [target_url, Url.normalize(target_url, base_url: @base_url)]
20
+ end
21
+ normalized_urls = normalized_by_url.values.compact.uniq
22
+ missing_urls = []
23
+
24
+ normalized_urls.each do |normalized_url|
25
+ next if @targets.key?(normalized_url)
26
+
27
+ resolved = resolved_page(normalized_url)
28
+ if resolved
29
+ @targets[normalized_url] = resolved
30
+ else
31
+ missing_urls << normalized_url
32
+ end
33
+ end
34
+
35
+ fetched_pages(missing_urls).each do |page|
36
+ normalized_url = Url.normalize(page.url, base_url: @base_url)
37
+ cache_page(page)
38
+ @targets[normalized_url] = resolution(page, normalized_url, crawled: false)
39
+ end
40
+
41
+ normalized_by_url.to_h { |target_url, normalized_url| [target_url, @targets[normalized_url]] }
42
+ end
43
+
44
+ private
45
+
46
+ def cache_page(page)
47
+ @pages[page.normalized_url] = page unless page.normalized_url.to_s.empty?
48
+ @pages[page.normalized_final_url] = page unless page.normalized_final_url.to_s.empty?
49
+ end
50
+
51
+ def fetched_pages(normalized_urls)
52
+ return [] if normalized_urls.empty?
53
+
54
+ @crawler.call(normalized_urls)
55
+ end
56
+
57
+ def resolved_page(normalized_url)
58
+ page = @pages[normalized_url]
59
+ resolution(page, normalized_url, crawled: true) if page
60
+ end
61
+
62
+ def resolution(page, normalized_url, crawled:)
63
+ {
64
+ crawled: crawled,
65
+ doc: page.doc,
66
+ error: page.error,
67
+ final_url: page.normalized_final_url || normalized_url,
68
+ headers: page.headers,
69
+ html: page.html?,
70
+ status: page.status
71
+ }
72
+ end
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Documents
5
+ class Canonical
6
+ attr_reader :href
7
+
8
+ def initialize(doc:, base_url:)
9
+ @href = doc&.at_css('link[rel="canonical"]')&.[]("href").to_s.strip
10
+ @base_url = base_url
11
+ end
12
+
13
+ def present?
14
+ !href.empty?
15
+ end
16
+
17
+ def url
18
+ Url.normalize(href, base_url: @base_url) if present?
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Crawlscope
4
+ module Documents
5
+ class Robots
6
+ ROBOTS_META_SELECTOR = 'meta[name="robots"], meta[name="googlebot"]'
7
+ X_ROBOTS_TAG_HEADER = "x-robots-tag"
8
+
9
+ def self.noindex_header?(headers)
10
+ noindex?(header_value(headers, X_ROBOTS_TAG_HEADER))
11
+ end
12
+
13
+ def self.noindex_meta?(doc)
14
+ return false unless doc
15
+
16
+ doc.css(ROBOTS_META_SELECTOR).any? { |tag| noindex?(tag["content"].to_s) }
17
+ end
18
+
19
+ def self.header_value(headers, name)
20
+ headers.find { |key, _value| key.to_s.casecmp?(name) }&.last.to_s
21
+ end
22
+
23
+ def self.directives(value)
24
+ value
25
+ .split(",")
26
+ .map { |directive| directive.split(":", 2).last.to_s.strip }
27
+ .reject(&:empty?)
28
+ end
29
+
30
+ def self.noindex?(value)
31
+ new(value).noindex?
32
+ end
33
+
34
+ def initialize(value)
35
+ @directives = self.class.directives(value)
36
+ end
37
+
38
+ def noindex?
39
+ include?("noindex") || include?("none")
40
+ end
41
+
42
+ def nofollow?
43
+ include?("nofollow") || include?("none")
44
+ end
45
+
46
+ def follow?
47
+ include?("follow")
48
+ end
49
+
50
+ private
51
+
52
+ def include?(value)
53
+ @directives.any? { |directive| directive.casecmp?(value) }
54
+ end
55
+ end
56
+ end
57
+ end
@@ -24,6 +24,10 @@ module Crawlscope
24
24
  headers.find { |key, _value| key.to_s.casecmp?(name.to_s) }&.last
25
25
  end
26
26
 
27
+ def canonical
28
+ Documents::Canonical.new(doc: doc, base_url: url)
29
+ end
30
+
27
31
  def server_timing
28
32
  @server_timing ||= ServerTiming.new(header("server-timing"))
29
33
  end