crawlscope 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/UPGRADE.md +76 -0
- data/docs/architecture.md +73 -0
- data/docs/rules.md +49 -0
- data/lib/crawlscope/configuration.rb +2 -1
- data/lib/crawlscope/crawl.rb +11 -79
- data/lib/crawlscope/crawling/fetcher_factory.rb +31 -0
- data/lib/crawlscope/crawling/target.rb +45 -0
- data/lib/crawlscope/crawling/target_resolver.rb +75 -0
- data/lib/crawlscope/documents/canonical.rb +22 -0
- data/lib/crawlscope/documents/robots.rb +57 -0
- data/lib/crawlscope/page.rb +4 -0
- data/lib/crawlscope/reporter.rb +6 -160
- data/lib/crawlscope/reporting/issue_presenter.rb +162 -0
- data/lib/crawlscope/reporting/url_formatter.rb +29 -0
- data/lib/crawlscope/rule_registry.rb +2 -2
- data/lib/crawlscope/rules/content_quality.rb +18 -0
- data/lib/crawlscope/rules/indexability.rb +9 -14
- data/lib/crawlscope/rules/links/canonical_checks.rb +102 -0
- data/lib/crawlscope/rules/links/extractor.rb +100 -0
- data/lib/crawlscope/rules/links/graph.rb +39 -0
- data/lib/crawlscope/rules/links/inbound_checks.rb +124 -0
- data/lib/crawlscope/rules/links/outgoing_checks.rb +104 -0
- data/lib/crawlscope/rules/links/resolution_check.rb +88 -0
- data/lib/crawlscope/rules/links/scope.rb +23 -0
- data/lib/crawlscope/rules/links/target_lookup.rb +38 -0
- data/lib/crawlscope/rules/links/url_checks.rb +54 -0
- data/lib/crawlscope/rules/links.rb +18 -600
- data/lib/crawlscope/rules/metadata.rb +2 -2
- data/lib/crawlscope/rules/structured_data.rb +26 -77
- data/lib/crawlscope/rules/uniqueness.rb +1 -1
- data/lib/crawlscope/schemas.rb +10 -2
- data/lib/crawlscope/server_timing/reporter.rb +2 -13
- data/lib/crawlscope/structured_data/audit.rb +17 -76
- data/lib/crawlscope/structured_data/check.rb +2 -0
- data/lib/crawlscope/structured_data/document.rb +14 -1
- data/lib/crawlscope/structured_data/entry.rb +31 -0
- data/lib/crawlscope/structured_data/outcome.rb +11 -0
- data/lib/crawlscope/structured_data/report.rb +6 -14
- data/lib/crawlscope/structured_data/reporter.rb +6 -11
- data/lib/crawlscope/structured_data/validator.rb +64 -0
- data/lib/crawlscope/version.rb +1 -1
- data/test/crawlscope/cli_test.rb +21 -0
- data/test/crawlscope/content_quality_rule_test.rb +51 -0
- data/test/crawlscope/links_rule_test.rb +16 -0
- data/test/crawlscope/reporter_test.rb +12 -0
- data/test/crawlscope/schema_registry_test.rb +22 -0
- data/test/crawlscope/structured_data_audit_test.rb +60 -0
- data/test/crawlscope/structured_data_report_test.rb +32 -0
- data/test/crawlscope/structured_data_rule_test.rb +11 -1
- data/test/crawlscope/structured_data_validator_test.rb +24 -0
- metadata +22 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9f1e7d817608a6023e7230e5568408576f375733ff0e466ab0ae9119b72cae8a
|
|
4
|
+
data.tar.gz: 4c645137855354cdb4847e641d78969a2f6014698d142c7a446b5555d4cbfbbb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e341bfed169b466693ec2bf72aec7ca53bc4c3ebabbc5935491ecd523b2edfb594a1bc44e16b3c22499c4a97ac4a1b70c4f0f42012933f652e7ea4bd1248c301
|
|
7
|
+
data.tar.gz: 6421dbc9dc6e15882b08a61f1a73ee23f2cdd02dbfdfc87136be363d65ac844d852065b6ef1041908278008627ecc740f66e979fe9be7ec57cca6fa71f158b90
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,24 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.10.2] - 2026-11-15
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- unify validation and reporting responsibilities
|
|
14
|
+
|
|
15
|
+
- enforce required structured data types by URL
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- accept date-only review publication dates
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
8
26
|
## [0.10.1] - 2026-11-08
|
|
9
27
|
|
|
10
28
|
|
data/UPGRADE.md
CHANGED
|
@@ -37,6 +37,82 @@ in the README; the shared workflow does not inherit another gem's permission.
|
|
|
37
37
|
|
|
38
38
|
## Next Release
|
|
39
39
|
|
|
40
|
+
### Review publication dates
|
|
41
|
+
|
|
42
|
+
Standalone and nested reviews now accept a date (`2026-09-16`) or timestamp for
|
|
43
|
+
`datePublished`, matching [Schema.org's Date or DateTime contract](https://schema.org/datePublished).
|
|
44
|
+
Invalid calendar dates still fail validation. Hosts no longer need to invent a
|
|
45
|
+
time to validate a review whose source publishes only its date.
|
|
46
|
+
|
|
47
|
+
### Required structured-data types
|
|
48
|
+
|
|
49
|
+
Set `config.required_structured_data_types` to a callback accepting a URL and
|
|
50
|
+
returning the types that page must contain. Separate strings require all types;
|
|
51
|
+
nested arrays accept alternatives, such as `[["Article", "BlogPosting",
|
|
52
|
+
"NewsArticle"]]`. The default remains no additional type requirements.
|
|
53
|
+
See [the rule configuration](docs/rules.md#required-types-by-url) for examples.
|
|
54
|
+
|
|
55
|
+
The default sitemap rule and direct URL audits apply the same requirements.
|
|
56
|
+
Custom rule registries must pass the callback through `required_types:` when
|
|
57
|
+
constructing their structured-data rule. Custom configuration objects supplied
|
|
58
|
+
to `Run` or `StructuredData::Check` can expose `required_structured_data_types`
|
|
59
|
+
to opt in. Existing objects without that optional setting retain their behavior.
|
|
60
|
+
|
|
61
|
+
### Shared audit outcomes and internal architecture
|
|
62
|
+
|
|
63
|
+
Direct structured-data audits now fail when an HTML page has no structured data
|
|
64
|
+
or contains a JSON-LD or microdata item missing `@type`. Detailed output,
|
|
65
|
+
summaries, and CLI/Rake exit status agree on these failures. Summary categories
|
|
66
|
+
also include failures on successful responses other than HTTP 200 and browser
|
|
67
|
+
responses without a status. Non-HTML responses remain successful skips.
|
|
68
|
+
|
|
69
|
+
Hosts using `validate_json_ld` or `crawlscope ldjson` must add the missing markup
|
|
70
|
+
or type to pass validation. Summary URL rows now retain the full URL, including
|
|
71
|
+
the host and query string; update text parsers that expected path-only rows.
|
|
72
|
+
The JSON report retains its existing shape and includes missing-type errors in
|
|
73
|
+
`validation_errors`.
|
|
74
|
+
|
|
75
|
+
JobPosting count checks are now opt-in and scoped to a host-supplied path
|
|
76
|
+
pattern. Crawlscope no longer assumes `/careers/...` identifies job detail
|
|
77
|
+
pages or that multiple JobPosting items are invalid on other pages. Hosts that
|
|
78
|
+
need exactly one JobPosting on job detail pages can replace the default rule:
|
|
79
|
+
|
|
80
|
+
```ruby
|
|
81
|
+
Crawlscope.configure do |config|
|
|
82
|
+
config.rule_registry = -> {
|
|
83
|
+
registry = Crawlscope::RuleRegistry.default(site_name: config.site_name)
|
|
84
|
+
rules = registry.rules.map do |rule|
|
|
85
|
+
if rule.code == :structured_data
|
|
86
|
+
Crawlscope::Rules::StructuredData.new(job_posting_path: %r{/careers/[^/]+/?\z})
|
|
87
|
+
else
|
|
88
|
+
rule
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
Crawlscope::RuleRegistry.new(rules: rules, default_codes: registry.default_codes)
|
|
92
|
+
}
|
|
93
|
+
end
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Use your application's actual job path pattern. The former
|
|
97
|
+
`Rules::StructuredData::CAREER_DETAIL_PATH` constant is removed; pass the pattern
|
|
98
|
+
directly when constructing the rule.
|
|
99
|
+
|
|
100
|
+
Existing `Crawl`, `Reporter`, rule entry points, issue fields, and
|
|
101
|
+
`StructuredData::Audit::Page` / `Outcome` constructors remain available.
|
|
102
|
+
Internal responsibilities now live under `Crawling`, `Documents`, `Reporting`,
|
|
103
|
+
and `Rules::Links`; see [architecture](docs/architecture.md) for extension points.
|
|
104
|
+
|
|
105
|
+
### Images missing alt attributes
|
|
106
|
+
|
|
107
|
+
The default `content_quality` rule now emits one `missing_image_alt` warning per
|
|
108
|
+
HTML page containing `<img>` elements without an `alt` attribute. Empty
|
|
109
|
+
`alt=""` attributes remain valid for decorative images. Details include `count`
|
|
110
|
+
and `image_sources`; text reports list the affected image sources.
|
|
111
|
+
|
|
112
|
+
No host configuration changes are required. Applications that count warnings or
|
|
113
|
+
fail crawls on warnings should expect this additional check. Add appropriate alt
|
|
114
|
+
text to informative images or `alt=""` to decorative images to resolve warnings.
|
|
115
|
+
|
|
40
116
|
### Query links retain their own URL identity
|
|
41
117
|
|
|
42
118
|
URL normalization now preserves query strings and still removes fragments.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
Crawlscope separates fetching, document interpretation, audit policy, and
|
|
4
|
+
presentation. Public entry points remain small coordinators. The directories
|
|
5
|
+
below contain the collaborators they use:
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
lib/crawlscope/
|
|
9
|
+
crawling/
|
|
10
|
+
fetcher_factory.rb HTTP/browser construction and browser load errors
|
|
11
|
+
target_resolver.rb URL normalization, page reuse, and target caching
|
|
12
|
+
target.rb Resolved target status and indexability
|
|
13
|
+
documents/
|
|
14
|
+
canonical.rb Canonical href extraction and URL resolution
|
|
15
|
+
robots.rb Shared robots directive interpretation
|
|
16
|
+
reporting/
|
|
17
|
+
issue_presenter.rb Issue headings, rows, evidence, and measurements
|
|
18
|
+
url_formatter.rb URL display that preserves origin and query identity
|
|
19
|
+
rules/
|
|
20
|
+
links.rb Public links rule; assembles one invocation
|
|
21
|
+
links/
|
|
22
|
+
extractor.rb Internal anchor extraction
|
|
23
|
+
scope.rb Shared link inclusion policy
|
|
24
|
+
target_lookup.rb Per-invocation target lookup and batch resolution
|
|
25
|
+
resolution_check.rb Broken and redirected link checks
|
|
26
|
+
graph.rb Inbound counts and source URLs
|
|
27
|
+
inbound_checks.rb Incoming-link policies
|
|
28
|
+
outgoing_checks.rb Outgoing-link and discovered sitemap policies
|
|
29
|
+
canonical_checks.rb Canonical target status and inbound-link policies
|
|
30
|
+
url_checks.rb URL hygiene
|
|
31
|
+
structured_data/
|
|
32
|
+
document.rb JSON-LD and microdata extraction
|
|
33
|
+
validator.rb Shared parse, type-presence, and schema validation
|
|
34
|
+
entry.rb Per-URL outcome and failure classification
|
|
35
|
+
outcome.rb Overall success derived from entries
|
|
36
|
+
audit.rb Fetches direct URLs and adapts validation findings
|
|
37
|
+
report.rb Summary selection and JSON serialization
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`Configuration` selects rules and runtime settings. `Run` invokes an audit and
|
|
41
|
+
its reporter. `Crawl` coordinates sitemap expansion, fetching, response checks,
|
|
42
|
+
and rule execution; `Crawling::TargetResolver` supplies rules with cached target
|
|
43
|
+
resolution through the existing `Context` protocol.
|
|
44
|
+
|
|
45
|
+
The links rule creates its collaborators for each call. A reused rule instance
|
|
46
|
+
does not retain a previous crawl's resolver or target cache. `Graph` indexes
|
|
47
|
+
resolved incoming links once, excludes self-links and noindex targets, and
|
|
48
|
+
retains full URL identity. Inbound and canonical checks ask the same graph for
|
|
49
|
+
counts instead of maintaining separate counting algorithms. Path inclusion
|
|
50
|
+
policy lives in `Scope`.
|
|
51
|
+
|
|
52
|
+
Both structured-data entry points use `StructuredData::Validator`. It returns
|
|
53
|
+
items and findings without writing output or choosing command exit status.
|
|
54
|
+
`Rules::StructuredData` translates findings into the sitemap audit's existing
|
|
55
|
+
warning issues. `StructuredData::Audit` translates them into direct-URL entries.
|
|
56
|
+
Host path requirements belong in explicit rule configuration.
|
|
57
|
+
|
|
58
|
+
`StructuredData::Entry` owns failure classification: fetch failure, missing
|
|
59
|
+
markup, validation failure, or success. `Outcome#ok?` and summary categories use
|
|
60
|
+
that classification. The legacy `Audit::Page` and `Audit::Outcome` constants are
|
|
61
|
+
aliases, preserving their constructors and fields.
|
|
62
|
+
|
|
63
|
+
`Reporter` owns text layout and ordering; `Reporting::IssuePresenter` formats
|
|
64
|
+
one issue without performing I/O. `Reporting::UrlFormatter` is shared by crawl
|
|
65
|
+
and timing reports. Structured-data errors use the same `ErrorFormatter` in
|
|
66
|
+
both audit reports. Existing `Issue#message` values remain available to API
|
|
67
|
+
consumers; presenters use structured evidence for richer text rows.
|
|
68
|
+
|
|
69
|
+
When adding a check, put document interpretation in a shared document object,
|
|
70
|
+
policy in a rule, and text formatting in a presenter. Keep network access behind
|
|
71
|
+
the fetcher and resolver interfaces. Add behavior tests at the public rule or
|
|
72
|
+
command boundary; use direct collaborator tests for policies that need their
|
|
73
|
+
own examples. Run `bundle exec rake verify` before handing off changes.
|
data/docs/rules.md
CHANGED
|
@@ -40,6 +40,53 @@ Checks:
|
|
|
40
40
|
- schema validation failures from the configured registry
|
|
41
41
|
- direct URL structured-data audits through `crawlscope:validate:ldjson`
|
|
42
42
|
|
|
43
|
+
Sitemap and direct URL audits share the same document validator, including
|
|
44
|
+
missing `@type` checks. Direct URL audits treat missing markup as a failure;
|
|
45
|
+
sitemap audits retain their warning severity for structured-data findings.
|
|
46
|
+
|
|
47
|
+
### Required types by URL
|
|
48
|
+
|
|
49
|
+
Schema validation checks the markup a page contains. To detect a missing type
|
|
50
|
+
even when other valid markup exists, configure a URL policy:
|
|
51
|
+
|
|
52
|
+
```ruby
|
|
53
|
+
Crawlscope.configure do |config|
|
|
54
|
+
config.required_structured_data_types = ->(url) {
|
|
55
|
+
case URI(url).path
|
|
56
|
+
when %r{\A/guides/[^/]+/?\z}
|
|
57
|
+
[["Article", "BlogPosting", "NewsArticle"]]
|
|
58
|
+
when "/guides"
|
|
59
|
+
["CollectionPage", "ItemList"]
|
|
60
|
+
when %r{/benchmark/?\z}
|
|
61
|
+
["WebPage", "Dataset"]
|
|
62
|
+
else
|
|
63
|
+
[]
|
|
64
|
+
end
|
|
65
|
+
}
|
|
66
|
+
end
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Each string requires that type. A nested array accepts any of its types.
|
|
70
|
+
Return an empty array for URLs without a type requirement. The callback receives
|
|
71
|
+
the full URL; the host owns the route policy and content classification.
|
|
72
|
+
|
|
73
|
+
The default sitemap rule and direct URL audits use this policy. A missing
|
|
74
|
+
requirement produces `structured_data_missing_required_type` in sitemap
|
|
75
|
+
findings and a validation failure in direct audits, including their reports and
|
|
76
|
+
exit status. Top-level items, JSON-LD type arrays and `@graph` nodes count;
|
|
77
|
+
an article linked inside an ItemList does not satisfy an article-page requirement.
|
|
78
|
+
Microdata items also count. Custom rule registries must pass the callback to
|
|
79
|
+
`Rules::StructuredData.new(required_types: callback)` themselves.
|
|
80
|
+
|
|
81
|
+
This checks type presence; the schema registry still validates supported types.
|
|
82
|
+
An index, product page or dataset should not acquire Article markup solely
|
|
83
|
+
because an external crawler classified its prose as an article.
|
|
84
|
+
|
|
85
|
+
Job detail pages can opt into an exactly-one-JobPosting policy with
|
|
86
|
+
`Rules::StructuredData.new(job_posting_path: your_path_pattern)` in a custom
|
|
87
|
+
rule registry. No host route patterns are assumed by default. See
|
|
88
|
+
[UPGRADE](../UPGRADE.md) for an example registry configuration.
|
|
89
|
+
|
|
43
90
|
## Uniqueness
|
|
44
91
|
|
|
45
92
|
Checks:
|
|
@@ -57,6 +104,8 @@ For larger crawls, exact duplicate checks still run and Crawlscope reports
|
|
|
57
104
|
|
|
58
105
|
Checks:
|
|
59
106
|
|
|
107
|
+
- missing `alt` attributes on `<img>` elements throughout the page, including
|
|
108
|
+
headers and footers (empty `alt=""` is allowed for decorative images)
|
|
60
109
|
- thin visible text
|
|
61
110
|
- low visible-text-to-HTML ratio
|
|
62
111
|
- low unique-token ratio
|
|
@@ -12,6 +12,7 @@ module Crawlscope
|
|
|
12
12
|
DEFAULT_TIMEOUT_SECONDS = 20
|
|
13
13
|
|
|
14
14
|
attr_writer :allowed_statuses, :base_url, :browser_factory, :concurrency, :fetch_executor, :network_idle_timeout_seconds, :output, :profile_token, :renderer, :rule_registry, :schema_registry, :scroll_page, :site_name, :sitemap_path, :timeout_seconds
|
|
15
|
+
attr_accessor :required_structured_data_types
|
|
15
16
|
|
|
16
17
|
def allowed_statuses
|
|
17
18
|
value = resolve(@allowed_statuses)
|
|
@@ -78,7 +79,7 @@ module Crawlscope
|
|
|
78
79
|
value = resolve(@rule_registry)
|
|
79
80
|
return value unless value.nil?
|
|
80
81
|
|
|
81
|
-
RuleRegistry.default(site_name: site_name)
|
|
82
|
+
RuleRegistry.default(site_name: site_name, required_types: required_structured_data_types)
|
|
82
83
|
end
|
|
83
84
|
|
|
84
85
|
def audit(base_url: self.base_url, sitemap_path: self.sitemap_path, rule_names: nil)
|
data/lib/crawlscope/crawl.rb
CHANGED
|
@@ -28,7 +28,10 @@ module Crawlscope
|
|
|
28
28
|
issues = IssueCollection.new
|
|
29
29
|
|
|
30
30
|
collect(pages, issues)
|
|
31
|
-
|
|
31
|
+
@target_resolver = Crawling::TargetResolver.new(
|
|
32
|
+
base_url: @base_url, pages: pages,
|
|
33
|
+
crawler: Crawler.new(page_fetcher: @page_fetcher, concurrency: @concurrency, fetch_executor: @fetch_executor)
|
|
34
|
+
)
|
|
32
35
|
scan(urls, pages, issues)
|
|
33
36
|
|
|
34
37
|
Result.new(
|
|
@@ -58,24 +61,13 @@ module Crawlscope
|
|
|
58
61
|
urls
|
|
59
62
|
end
|
|
60
63
|
|
|
61
|
-
def browser
|
|
62
|
-
Browser.new(
|
|
63
|
-
base_url: @base_url,
|
|
64
|
-
timeout_seconds: @timeout_seconds,
|
|
65
|
-
network_idle_timeout_seconds: @network_idle_timeout_seconds,
|
|
66
|
-
profile_token: @profile_token,
|
|
67
|
-
scroll_page: @scroll_page
|
|
68
|
-
)
|
|
69
|
-
rescue LoadError => error
|
|
70
|
-
raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
|
|
71
|
-
end
|
|
72
|
-
|
|
73
64
|
def page
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
65
|
+
Crawling::FetcherFactory.new(
|
|
66
|
+
base_url: @base_url, renderer: @renderer, timeout_seconds: @timeout_seconds,
|
|
67
|
+
browser_factory: @browser_factory, profile_token: @profile_token,
|
|
68
|
+
network_idle_timeout_seconds: @network_idle_timeout_seconds,
|
|
69
|
+
scroll_page: @scroll_page, adapter: http_adapter
|
|
70
|
+
).call
|
|
79
71
|
end
|
|
80
72
|
|
|
81
73
|
def http_adapter
|
|
@@ -119,20 +111,6 @@ module Crawlscope
|
|
|
119
111
|
end
|
|
120
112
|
end
|
|
121
113
|
|
|
122
|
-
def cache(pages)
|
|
123
|
-
@pages = {}
|
|
124
|
-
@targets = {}
|
|
125
|
-
|
|
126
|
-
pages.each do |page|
|
|
127
|
-
cache_page(page)
|
|
128
|
-
end
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
def cache_page(page)
|
|
132
|
-
@pages[page.normalized_url] = page unless page.normalized_url.to_s.empty?
|
|
133
|
-
@pages[page.normalized_final_url] = page unless page.normalized_final_url.to_s.empty?
|
|
134
|
-
end
|
|
135
|
-
|
|
136
114
|
def scan(urls, pages, issues)
|
|
137
115
|
@rules.each do |rule|
|
|
138
116
|
rule.call(urls: urls, pages: pages, issues: issues, context: context)
|
|
@@ -144,53 +122,7 @@ module Crawlscope
|
|
|
144
122
|
end
|
|
145
123
|
|
|
146
124
|
def resolve_all(target_urls)
|
|
147
|
-
|
|
148
|
-
[target_url, Url.normalize(target_url, base_url: @base_url)]
|
|
149
|
-
end
|
|
150
|
-
normalized_urls = normalized_by_url.values.compact.uniq
|
|
151
|
-
missing_urls = []
|
|
152
|
-
|
|
153
|
-
normalized_urls.each do |normalized_url|
|
|
154
|
-
next if @targets.key?(normalized_url)
|
|
155
|
-
|
|
156
|
-
resolved = resolved_page(normalized_url)
|
|
157
|
-
if resolved
|
|
158
|
-
@targets[normalized_url] = resolved
|
|
159
|
-
else
|
|
160
|
-
missing_urls << normalized_url
|
|
161
|
-
end
|
|
162
|
-
end
|
|
163
|
-
|
|
164
|
-
fetched_pages(missing_urls).each do |page|
|
|
165
|
-
normalized_url = Url.normalize(page.url, base_url: @base_url)
|
|
166
|
-
cache_page(page)
|
|
167
|
-
@targets[normalized_url] = resolution(page, normalized_url, crawled: false)
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
normalized_by_url.to_h { |target_url, normalized_url| [target_url, @targets[normalized_url]] }
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
def fetched_pages(normalized_urls)
|
|
174
|
-
return [] if normalized_urls.empty?
|
|
175
|
-
|
|
176
|
-
Crawler.new(page_fetcher: @page_fetcher, concurrency: @concurrency, fetch_executor: @fetch_executor).call(normalized_urls)
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
def resolved_page(normalized_url)
|
|
180
|
-
page = @pages[normalized_url]
|
|
181
|
-
resolution(page, normalized_url, crawled: true) if page
|
|
182
|
-
end
|
|
183
|
-
|
|
184
|
-
def resolution(page, normalized_url, crawled:)
|
|
185
|
-
{
|
|
186
|
-
crawled: crawled,
|
|
187
|
-
doc: page.doc,
|
|
188
|
-
error: page.error,
|
|
189
|
-
final_url: page.normalized_final_url || normalized_url,
|
|
190
|
-
headers: page.headers,
|
|
191
|
-
html: page.html?,
|
|
192
|
-
status: page.status
|
|
193
|
-
}
|
|
125
|
+
@target_resolver.call(target_urls)
|
|
194
126
|
end
|
|
195
127
|
|
|
196
128
|
def redirected?(page)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Crawling
|
|
5
|
+
class FetcherFactory
|
|
6
|
+
def initialize(base_url:, renderer:, timeout_seconds:, browser_factory: nil,
|
|
7
|
+
network_idle_timeout_seconds: Configuration::DEFAULT_BROWSER_NETWORK_IDLE_TIMEOUT_SECONDS,
|
|
8
|
+
scroll_page: Configuration::DEFAULT_BROWSER_SCROLL_PAGE, profile_token: nil, adapter: nil)
|
|
9
|
+
@base_url = base_url
|
|
10
|
+
@renderer = renderer
|
|
11
|
+
@timeout_seconds = timeout_seconds
|
|
12
|
+
@browser_factory = browser_factory
|
|
13
|
+
@network_idle_timeout_seconds = network_idle_timeout_seconds
|
|
14
|
+
@scroll_page = scroll_page
|
|
15
|
+
@profile_token = profile_token
|
|
16
|
+
@adapter = adapter
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def call
|
|
20
|
+
return Http.new(base_url: @base_url, timeout_seconds: @timeout_seconds, adapter: @adapter, profile_token: @profile_token) unless @renderer == :browser
|
|
21
|
+
return @browser_factory.call if @browser_factory
|
|
22
|
+
|
|
23
|
+
Browser.new(base_url: @base_url, timeout_seconds: @timeout_seconds,
|
|
24
|
+
network_idle_timeout_seconds: @network_idle_timeout_seconds,
|
|
25
|
+
scroll_page: @scroll_page, profile_token: @profile_token)
|
|
26
|
+
rescue LoadError => error
|
|
27
|
+
raise ConfigurationError, "Browser rendering requires the ferrum gem (#{error.message})"
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Crawling
|
|
5
|
+
Target = Data.define(:target_url, :resolution) do
|
|
6
|
+
def allowed?(statuses)
|
|
7
|
+
statuses.include?(status)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def final_path
|
|
11
|
+
Url.path(final_url)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def final_url
|
|
15
|
+
value = resolution[:final_url].to_s
|
|
16
|
+
value.empty? ? target_url : value
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def ignored_error?
|
|
20
|
+
resolution && status.nil? && resolution[:crawled] && resolution[:error]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def html?
|
|
24
|
+
resolution && resolution[:html]
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def noindex?
|
|
28
|
+
Documents::Robots.noindex_header?(resolution[:headers] || {}) ||
|
|
29
|
+
Documents::Robots.noindex_meta?(resolution[:doc])
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def status
|
|
33
|
+
resolution && resolution[:status]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def redirect?
|
|
37
|
+
(status && (300..399).cover?(status.to_i)) || final_url != target_url
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def unresolved?
|
|
41
|
+
resolution.nil? || (status.nil? && !ignored_error?)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Crawling
|
|
5
|
+
class TargetResolver
|
|
6
|
+
def initialize(base_url:, crawler:, pages:)
|
|
7
|
+
@base_url = base_url
|
|
8
|
+
@crawler = crawler
|
|
9
|
+
@pages = {}
|
|
10
|
+
@targets = {}
|
|
11
|
+
|
|
12
|
+
pages.each do |page|
|
|
13
|
+
cache_page(page)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call(target_urls)
|
|
18
|
+
normalized_by_url = Array(target_urls).to_h do |target_url|
|
|
19
|
+
[target_url, Url.normalize(target_url, base_url: @base_url)]
|
|
20
|
+
end
|
|
21
|
+
normalized_urls = normalized_by_url.values.compact.uniq
|
|
22
|
+
missing_urls = []
|
|
23
|
+
|
|
24
|
+
normalized_urls.each do |normalized_url|
|
|
25
|
+
next if @targets.key?(normalized_url)
|
|
26
|
+
|
|
27
|
+
resolved = resolved_page(normalized_url)
|
|
28
|
+
if resolved
|
|
29
|
+
@targets[normalized_url] = resolved
|
|
30
|
+
else
|
|
31
|
+
missing_urls << normalized_url
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
fetched_pages(missing_urls).each do |page|
|
|
36
|
+
normalized_url = Url.normalize(page.url, base_url: @base_url)
|
|
37
|
+
cache_page(page)
|
|
38
|
+
@targets[normalized_url] = resolution(page, normalized_url, crawled: false)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
normalized_by_url.to_h { |target_url, normalized_url| [target_url, @targets[normalized_url]] }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def cache_page(page)
|
|
47
|
+
@pages[page.normalized_url] = page unless page.normalized_url.to_s.empty?
|
|
48
|
+
@pages[page.normalized_final_url] = page unless page.normalized_final_url.to_s.empty?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def fetched_pages(normalized_urls)
|
|
52
|
+
return [] if normalized_urls.empty?
|
|
53
|
+
|
|
54
|
+
@crawler.call(normalized_urls)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def resolved_page(normalized_url)
|
|
58
|
+
page = @pages[normalized_url]
|
|
59
|
+
resolution(page, normalized_url, crawled: true) if page
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def resolution(page, normalized_url, crawled:)
|
|
63
|
+
{
|
|
64
|
+
crawled: crawled,
|
|
65
|
+
doc: page.doc,
|
|
66
|
+
error: page.error,
|
|
67
|
+
final_url: page.normalized_final_url || normalized_url,
|
|
68
|
+
headers: page.headers,
|
|
69
|
+
html: page.html?,
|
|
70
|
+
status: page.status
|
|
71
|
+
}
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Documents
|
|
5
|
+
class Canonical
|
|
6
|
+
attr_reader :href
|
|
7
|
+
|
|
8
|
+
def initialize(doc:, base_url:)
|
|
9
|
+
@href = doc&.at_css('link[rel="canonical"]')&.[]("href").to_s.strip
|
|
10
|
+
@base_url = base_url
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def present?
|
|
14
|
+
!href.empty?
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def url
|
|
18
|
+
Url.normalize(href, base_url: @base_url) if present?
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Crawlscope
|
|
4
|
+
module Documents
|
|
5
|
+
class Robots
|
|
6
|
+
ROBOTS_META_SELECTOR = 'meta[name="robots"], meta[name="googlebot"]'
|
|
7
|
+
X_ROBOTS_TAG_HEADER = "x-robots-tag"
|
|
8
|
+
|
|
9
|
+
def self.noindex_header?(headers)
|
|
10
|
+
noindex?(header_value(headers, X_ROBOTS_TAG_HEADER))
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.noindex_meta?(doc)
|
|
14
|
+
return false unless doc
|
|
15
|
+
|
|
16
|
+
doc.css(ROBOTS_META_SELECTOR).any? { |tag| noindex?(tag["content"].to_s) }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.header_value(headers, name)
|
|
20
|
+
headers.find { |key, _value| key.to_s.casecmp?(name) }&.last.to_s
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.directives(value)
|
|
24
|
+
value
|
|
25
|
+
.split(",")
|
|
26
|
+
.map { |directive| directive.split(":", 2).last.to_s.strip }
|
|
27
|
+
.reject(&:empty?)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.noindex?(value)
|
|
31
|
+
new(value).noindex?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def initialize(value)
|
|
35
|
+
@directives = self.class.directives(value)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def noindex?
|
|
39
|
+
include?("noindex") || include?("none")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def nofollow?
|
|
43
|
+
include?("nofollow") || include?("none")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def follow?
|
|
47
|
+
include?("follow")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def include?(value)
|
|
53
|
+
@directives.any? { |directive| directive.casecmp?(value) }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
data/lib/crawlscope/page.rb
CHANGED
|
@@ -24,6 +24,10 @@ module Crawlscope
|
|
|
24
24
|
headers.find { |key, _value| key.to_s.casecmp?(name.to_s) }&.last
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
+
def canonical
|
|
28
|
+
Documents::Canonical.new(doc: doc, base_url: url)
|
|
29
|
+
end
|
|
30
|
+
|
|
27
31
|
def server_timing
|
|
28
32
|
@server_timing ||= ServerTiming.new(header("server-timing"))
|
|
29
33
|
end
|