url_canonicalize 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +67 -87
- data/README.md +141 -7
- data/SECURITY.md +25 -0
- data/lib/monkey_patches/addressable/uri.rb +2 -2
- data/lib/monkey_patches/string.rb +2 -2
- data/lib/monkey_patches/uri.rb +2 -2
- data/lib/url_canonicalize/bounded_body.rb +23 -0
- data/lib/url_canonicalize/destination.rb +120 -0
- data/lib/url_canonicalize/exception.rb +3 -0
- data/lib/url_canonicalize/http.rb +81 -54
- data/lib/url_canonicalize/link_header.rb +34 -0
- data/lib/url_canonicalize/media_type.rb +23 -0
- data/lib/url_canonicalize/options.rb +79 -0
- data/lib/url_canonicalize/request.rb +79 -68
- data/lib/url_canonicalize/response.rb +7 -4
- data/lib/url_canonicalize/uri.rb +7 -10
- data/lib/url_canonicalize/version.rb +1 -1
- data/lib/url_canonicalize.rb +10 -5
- metadata +14 -21
- data/.codeclimate.yml +0 -25
- data/.github/workflows/codeql-analysis.yml +0 -70
- data/.gitignore +0 -53
- data/.hound.yml +0 -4
- data/.rspec +0 -5
- data/.rubocop.yml +0 -61
- data/.ruby-gemset +0 -1
- data/.travis.yml +0 -8
- data/Gemfile +0 -22
- data/Gemfile.lock +0 -110
- data/Guardfile +0 -18
- data/Rakefile +0 -3
- data/circle.yml +0 -29
- data/renovate.json +0 -6
- data/url_canonicalize.gemspec +0 -29
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6b8529a2f607ea6c3be592b855fa12bb79256bc62a11dfb94aa7b2d6aec1b45f
|
|
4
|
+
data.tar.gz: 54a13bd40248b52cab8b3058fe04216bef20598896431cd90dd7e4cb4089d67f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 76310623c993c4550972e0b08bb28516c1673d246f3b0173738fbdb8e65f416d6375b7174d83e513d640a7c71c56de06fc115e2e91bb37a3d1e583c12a0231a6
|
|
7
|
+
data.tar.gz: a68a7ee377c1cf90d162ee44a76c54a01936e0e1ace7055c0cd084a89bf3ade561060651aed21231f23a8e4d0cc129384534d9b9ced4c7dffb7e2a0bbaef7d9d
|
data/CHANGELOG.md
CHANGED
|
@@ -1,117 +1,97 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
4
|
|
|
5
|
-
[
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to
|
|
7
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
8
|
|
|
7
|
-
|
|
9
|
+
## [Unreleased]
|
|
8
10
|
|
|
9
|
-
|
|
11
|
+
## [1.0.2] - 2026-07-24
|
|
10
12
|
|
|
11
|
-
|
|
13
|
+
### Fixed
|
|
12
14
|
|
|
13
|
-
-
|
|
15
|
+
- Add `rake` to the development bundle so the `rake build` release check can
|
|
16
|
+
run ([#204](https://github.com/dominicsayers/url_canonicalize/pull/204))
|
|
14
17
|
|
|
15
|
-
|
|
18
|
+
### Changed
|
|
16
19
|
|
|
17
|
-
|
|
20
|
+
- Announce each step of `bin/release/prepare` as it starts so the command
|
|
21
|
+
shows progress instead of appearing to hang
|
|
22
|
+
([#206](https://github.com/dominicsayers/url_canonicalize/pull/206))
|
|
23
|
+
- Update the `ruby/setup-ruby` CI action to v1.321.0
|
|
24
|
+
([#205](https://github.com/dominicsayers/url_canonicalize/pull/205))
|
|
18
25
|
|
|
19
|
-
## [
|
|
26
|
+
## [1.0.1] - 2026-07-22
|
|
20
27
|
|
|
21
|
-
|
|
28
|
+
### Security
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
- Verify TLS certificates and hostnames, block unsafe destinations at every
|
|
31
|
+
hop, pin DNS results, restrict ports, bound response bodies and enforce an
|
|
32
|
+
overall deadline
|
|
33
|
+
([#184](https://github.com/dominicsayers/url_canonicalize/issues/184))
|
|
24
34
|
|
|
25
|
-
|
|
35
|
+
### Fixed
|
|
26
36
|
|
|
27
|
-
|
|
37
|
+
- Correct the redirect and canonical-link HTTP state machine: follow all
|
|
38
|
+
redirect responses consistently, bound redirect and canonical-link chains
|
|
39
|
+
with one visited-URL set and one hop budget, parse `Link` headers properly,
|
|
40
|
+
resolve relative references per RFC 3986, match HTML canonical links
|
|
41
|
+
case-insensitively against the document base URL, replace host-specific HEAD
|
|
42
|
+
behavior with a generic HEAD-to-GET fallback and preserve underlying error
|
|
43
|
+
causes ([#185](https://github.com/dominicsayers/url_canonicalize/issues/185))
|
|
28
44
|
|
|
29
|
-
|
|
45
|
+
### Changed
|
|
30
46
|
|
|
31
|
-
|
|
47
|
+
- Refresh the test and coverage strategy: replace `coveralls-ruby` with a
|
|
48
|
+
current SimpleCov release, enforce 100% line and branch coverage on CI,
|
|
49
|
+
test redirect and canonical-link behavior deterministically at the HTTP
|
|
50
|
+
boundary and isolate environment changes in every example
|
|
51
|
+
([#189](https://github.com/dominicsayers/url_canonicalize/issues/189))
|
|
52
|
+
- Refresh gem packaging and release automation: package a deterministic
|
|
53
|
+
whitelist of runtime files, verify the built gem in CI, add full RubyGems
|
|
54
|
+
metadata, publish through RubyGems Trusted Publishing from a tag-triggered
|
|
55
|
+
workflow, and document security reporting and release procedures
|
|
56
|
+
([#190](https://github.com/dominicsayers/url_canonicalize/issues/190))
|
|
32
57
|
|
|
33
|
-
[
|
|
58
|
+
## [1.0.0] - 2024-01-27
|
|
34
59
|
|
|
35
|
-
|
|
60
|
+
### Changed
|
|
36
61
|
|
|
37
|
-
|
|
62
|
+
- Require Ruby 3.1 or later
|
|
63
|
+
- Adapt to the behavior of current `uri` gem releases
|
|
64
|
+
- Update development dependencies and CI configuration
|
|
38
65
|
|
|
39
|
-
|
|
66
|
+
## [0.2.1] - 2022-02-01
|
|
40
67
|
|
|
41
|
-
|
|
68
|
+
### Changed
|
|
42
69
|
|
|
43
|
-
|
|
70
|
+
- Require a recent Nokogiri version
|
|
71
|
+
- Update development dependencies, RuboCop configuration and CI configuration
|
|
44
72
|
|
|
45
|
-
[
|
|
73
|
+
## [0.2.0] - 2018-06-30
|
|
46
74
|
|
|
47
|
-
|
|
75
|
+
### Added
|
|
48
76
|
|
|
49
|
-
|
|
77
|
+
- Return a Nokogiri XML document from a successful response when requested
|
|
78
|
+
([#5](https://github.com/dominicsayers/url_canonicalize/issues/5))
|
|
50
79
|
|
|
51
|
-
|
|
80
|
+
### Changed
|
|
52
81
|
|
|
53
|
-
-
|
|
82
|
+
- Require a secure Nokogiri version
|
|
54
83
|
|
|
55
|
-
##
|
|
84
|
+
## 0.1.15 and earlier
|
|
56
85
|
|
|
57
|
-
|
|
86
|
+
Releases from 0.0.1 (2016-10-20) to 0.1.15 (2017-03-14) predate this
|
|
87
|
+
changelog format. Their history is available from the
|
|
88
|
+
[v0.1.15 tag](https://github.com/dominicsayers/url_canonicalize/releases/tag/v0.1.15)
|
|
89
|
+
and the
|
|
90
|
+
[commit log](https://github.com/dominicsayers/url_canonicalize/commits/v0.1.15).
|
|
58
91
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.5...v0.1.6)
|
|
66
|
-
|
|
67
|
-
## [v0.1.5](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.5) (2016-10-26)
|
|
68
|
-
|
|
69
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.4...v0.1.5)
|
|
70
|
-
|
|
71
|
-
## [v0.1.4](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.4) (2016-10-26)
|
|
72
|
-
|
|
73
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.2...v0.1.4)
|
|
74
|
-
|
|
75
|
-
## [v0.1.2](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.2) (2016-10-26)
|
|
76
|
-
|
|
77
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.1...v0.1.2)
|
|
78
|
-
|
|
79
|
-
## [v0.1.1](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.1) (2016-10-26)
|
|
80
|
-
|
|
81
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.0...v0.1.1)
|
|
82
|
-
|
|
83
|
-
## [v0.1.0](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.0) (2016-10-20)
|
|
84
|
-
|
|
85
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.7...v0.1.0)
|
|
86
|
-
|
|
87
|
-
## [v0.0.7](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.7) (2016-10-20)
|
|
88
|
-
|
|
89
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.6...v0.0.7)
|
|
90
|
-
|
|
91
|
-
## [v0.0.6](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.6) (2016-10-20)
|
|
92
|
-
|
|
93
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.5...v0.0.6)
|
|
94
|
-
|
|
95
|
-
## [v0.0.5](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.5) (2016-10-20)
|
|
96
|
-
|
|
97
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.4...v0.0.5)
|
|
98
|
-
|
|
99
|
-
## [v0.0.4](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.4) (2016-10-20)
|
|
100
|
-
|
|
101
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.3...v0.0.4)
|
|
102
|
-
|
|
103
|
-
## [v0.0.3](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.3) (2016-10-20)
|
|
104
|
-
|
|
105
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.2...v0.0.3)
|
|
106
|
-
|
|
107
|
-
## [v0.0.2](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.2) (2016-10-20)
|
|
108
|
-
|
|
109
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.1...v0.0.2)
|
|
110
|
-
|
|
111
|
-
## [v0.0.1](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.1) (2016-10-20)
|
|
112
|
-
|
|
113
|
-
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/fec855bec42813304f93c8ef87035f184b054344...v0.0.1)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
|
|
92
|
+
[Unreleased]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.2...HEAD
|
|
93
|
+
[1.0.2]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.1...v1.0.2
|
|
94
|
+
[1.0.1]: https://github.com/dominicsayers/url_canonicalize/compare/v1.0.0...v1.0.1
|
|
95
|
+
[1.0.0]: https://github.com/dominicsayers/url_canonicalize/compare/v0.2.1...v1.0.0
|
|
96
|
+
[0.2.1]: https://github.com/dominicsayers/url_canonicalize/compare/v0.2.0...v0.2.1
|
|
97
|
+
[0.2.0]: https://github.com/dominicsayers/url_canonicalize/compare/v0.1.15...v0.2.0
|
data/README.md
CHANGED
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
# URLCanonicalize
|
|
2
|
-
[](https://rubygems.org/gems/url_canonicalize)
|
|
3
3
|
[](https://rubygems.org/gems/url_canonicalize)
|
|
4
|
-
[](https://coveralls.io/github/dominicsayers/url_canonicalize?branch=master)
|
|
7
|
-
[](https://dependencyci.com/github/dominicsayers/url_canonicalize)
|
|
8
|
-
[](https://hakiri.io/github/dominicsayers/url_canonicalize/master)
|
|
9
|
-
[](https://www.codacy.com/app/dominicsayers/url_canonicalize)
|
|
4
|
+
[](https://dl.circleci.com/status-badge/redirect/gh/dominicsayers/url_canonicalize/tree/main)
|
|
5
|
+
[](https://github.com/dominicsayers/url_canonicalize/actions/workflows/codeql-analysis.yml)
|
|
10
6
|
|
|
11
7
|
URLCanonicalize is a Ruby gem that finds the canonical version of a URL. It
|
|
12
8
|
provides `canonicalize` methods for the String, URI::HTTP, URI::HTTPS and
|
|
13
9
|
Addressable::URI classes.
|
|
14
10
|
|
|
11
|
+
Ruby 3.1 or later is required.
|
|
12
|
+
|
|
15
13
|
## Installation
|
|
16
14
|
|
|
17
15
|
Add this line to your application's Gemfile:
|
|
@@ -22,6 +20,9 @@ gem 'url_canonicalize'
|
|
|
22
20
|
|
|
23
21
|
## Usage
|
|
24
22
|
|
|
23
|
+
Requiring the gem extends `String`, `URI::HTTP`, `URI::HTTPS` and
|
|
24
|
+
`Addressable::URI` with a `canonicalize` method:
|
|
25
|
+
|
|
25
26
|
```ruby
|
|
26
27
|
'http://www.twitter.com'.canonicalize # => 'https://twitter.com/'
|
|
27
28
|
|
|
@@ -30,6 +31,139 @@ URI('http://www.twitter.com').canonicalize # => #<URI::HTTP:0x00000008767908 URL
|
|
|
30
31
|
Addressable::URI.canonicalize('http://www.twitter.com') # => #<Addressable::URI:0x43c9 URI:https://twitter.com/>
|
|
31
32
|
```
|
|
32
33
|
|
|
34
|
+
Options can be passed to the module API or any of the `canonicalize` extension
|
|
35
|
+
methods:
|
|
36
|
+
|
|
37
|
+
```ruby
|
|
38
|
+
URLCanonicalize.fetch(
|
|
39
|
+
'https://example.com/article',
|
|
40
|
+
total_timeout: 10,
|
|
41
|
+
max_body_bytes: 512_000
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
'https://example.com/article'.canonicalize(total_timeout: 10)
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
If you prefer the module API without the core-class extensions, call
|
|
48
|
+
`URLCanonicalize.canonicalize(url)` (returns the canonical URL string) or
|
|
49
|
+
`URLCanonicalize.fetch(url)` (returns the full response object).
|
|
50
|
+
|
|
51
|
+
## Security and limits
|
|
52
|
+
|
|
53
|
+
URLCanonicalize treats every fetched URL as untrusted, including redirect and
|
|
54
|
+
`rel="canonical"` destinations. Before connecting, it resolves the host,
|
|
55
|
+
rejects the destination if any DNS answer is loopback, private, link-local,
|
|
56
|
+
multicast, unspecified, reserved or otherwise not publicly routable, and pins
|
|
57
|
+
the connection to the validated address. URLs containing user information are
|
|
58
|
+
rejected. HTTPS always uses peer certificate and hostname verification; there
|
|
59
|
+
is no option to disable TLS verification.
|
|
60
|
+
|
|
61
|
+
IPv6 validation is fail-closed: only prefixes currently allocated in IANA's
|
|
62
|
+
global-unicast registry are eligible, with special-purpose ranges excluded.
|
|
63
|
+
|
|
64
|
+
Environment HTTP proxies are deliberately disabled because proxy-side DNS
|
|
65
|
+
resolution would bypass destination validation. Ports `80` and `443` are the
|
|
66
|
+
only ports allowed by default.
|
|
67
|
+
|
|
68
|
+
The default resource limits are:
|
|
69
|
+
|
|
70
|
+
- 30 seconds for the complete canonicalization operation, across every hop;
|
|
71
|
+
- 8 seconds to open a connection, 15 seconds to read and 8 seconds to write;
|
|
72
|
+
- 1 MiB for a buffered response body;
|
|
73
|
+
- 10 followed hops, shared between redirects and followed canonical links.
|
|
74
|
+
|
|
75
|
+
Only successful GET responses with `text/html`, `application/xhtml+xml`,
|
|
76
|
+
`application/xml` or `text/xml` media types are buffered. Media-type casing and
|
|
77
|
+
parameters such as `charset` are accepted. Other response bodies are drained
|
|
78
|
+
without being retained, and only HTML/XHTML responses are parsed for canonical
|
|
79
|
+
link elements.
|
|
80
|
+
|
|
81
|
+
All limits can be made stricter, and additional ports can be explicitly
|
|
82
|
+
allowed:
|
|
83
|
+
|
|
84
|
+
```ruby
|
|
85
|
+
URLCanonicalize.fetch(
|
|
86
|
+
'https://example.com:8443/article',
|
|
87
|
+
allowed_ports: [443, 8443],
|
|
88
|
+
max_body_bytes: 256_000,
|
|
89
|
+
total_timeout: 5,
|
|
90
|
+
open_timeout: 2,
|
|
91
|
+
read_timeout: 3,
|
|
92
|
+
write_timeout: 2,
|
|
93
|
+
max_redirects: 5
|
|
94
|
+
)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Private-network fetching is available only as an explicit escape hatch for
|
|
98
|
+
trusted URLs:
|
|
99
|
+
|
|
100
|
+
```ruby
|
|
101
|
+
URLCanonicalize.fetch(
|
|
102
|
+
'http://intranet.example.test',
|
|
103
|
+
allow_private_networks: true
|
|
104
|
+
)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Enabling `allow_private_networks` relaxes the SSRF protection. It still pins the
|
|
108
|
+
resolved address and enforces the user-information and port policies.
|
|
109
|
+
|
|
110
|
+
Policy violations raise `URLCanonicalize::Exception::Security`, oversized
|
|
111
|
+
responses raise `URLCanonicalize::Exception::ResponseTooLarge`, and the overall
|
|
112
|
+
deadline raises `URLCanonicalize::Exception::Timeout`. Other socket and TLS
|
|
113
|
+
failures continue to surface as `URLCanonicalize::Exception::Failure`.
|
|
114
|
+
|
|
115
|
+
## How URLs are followed
|
|
116
|
+
|
|
117
|
+
Every hop starts with a `HEAD` request. If the response carries no canonical
|
|
118
|
+
hint in its headers, the request is retried as a `GET` so the HTML can be
|
|
119
|
+
inspected. A `HEAD` request rejected with `403 Forbidden`,
|
|
120
|
+
`405 Method Not Allowed` or `501 Not Implemented` is also retried once as a
|
|
121
|
+
`GET`, because some servers refuse `HEAD` requests outright; an unsuccessful
|
|
122
|
+
`GET` is not retried.
|
|
123
|
+
|
|
124
|
+
All redirect responses (`301`, `302`, `303`, `307` and `308`) are followed
|
|
125
|
+
through their `Location` header. A redirection without a usable `Location` is
|
|
126
|
+
treated as a failure.
|
|
127
|
+
|
|
128
|
+
Canonical URLs are discovered from `Link` response headers whose `rel` tokens
|
|
129
|
+
include `canonical` (RFC 8288), and from `<link rel="canonical">` elements in
|
|
130
|
+
the HTML `<head>`, matched case-insensitively. Header targets resolve against
|
|
131
|
+
the request URL; HTML targets resolve against the document base URL, honouring
|
|
132
|
+
any `<base href>` element. Absolute, protocol-relative, root-relative,
|
|
133
|
+
path-relative and query-only references are all resolved per RFC 3986, and
|
|
134
|
+
only `http` or `https` results are followed.
|
|
135
|
+
|
|
136
|
+
Redirects and followed canonical links share a single visited-URL set and a
|
|
137
|
+
single hop budget (`max_redirects`), so cycles and over-long chains always
|
|
138
|
+
terminate. A cycle or exhausted budget returns the last successfully fetched
|
|
139
|
+
response if there is one, and raises `URLCanonicalize::Exception::Redirect`
|
|
140
|
+
otherwise. When a failure is caused by an underlying exception, that exception
|
|
141
|
+
is preserved as the `cause` of the raised
|
|
142
|
+
`URLCanonicalize::Exception::Failure`.
|
|
143
|
+
|
|
144
|
+
## Errors
|
|
145
|
+
|
|
146
|
+
All errors raised by the gem are subclasses of `URLCanonicalize::Exception`,
|
|
147
|
+
so `rescue URLCanonicalize::Exception` catches everything below:
|
|
148
|
+
|
|
149
|
+
| Exception | Raised when |
|
|
150
|
+
| --------- | ----------- |
|
|
151
|
+
| `Exception::URI` | the URL cannot be parsed, or is not `http`/`https` |
|
|
152
|
+
| `Exception::Security` | a destination violates the SSRF, userinfo or port policy |
|
|
153
|
+
| `Exception::Redirect` | a redirect or canonical-link chain loops or exceeds `max_redirects` |
|
|
154
|
+
| `Exception::Timeout` | the operation exceeds `total_timeout` |
|
|
155
|
+
| `Exception::ResponseTooLarge` | a response body exceeds `max_body_bytes` |
|
|
156
|
+
| `Exception::Failure` | the request fails (network error, TLS failure or unsuccessful HTTP status) |
|
|
157
|
+
|
|
158
|
+
When a failure wraps an underlying exception, the original is available as the
|
|
159
|
+
raised error's `cause`.
|
|
160
|
+
|
|
161
|
+
## Contributing and security
|
|
162
|
+
|
|
163
|
+
Development and release instructions are in
|
|
164
|
+
[CONTRIBUTING.md](CONTRIBUTING.md). Please report vulnerabilities privately as
|
|
165
|
+
described in [SECURITY.md](SECURITY.md), not through public issues.
|
|
166
|
+
|
|
33
167
|
## More Information
|
|
34
168
|
|
|
35
169
|
URLCanonical follows HTTP redirects and also looks for `rel="canonical"` hints
|
data/SECURITY.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Supported versions
|
|
4
|
+
|
|
5
|
+
| Version | Supported |
|
|
6
|
+
| ------- | ------------------ |
|
|
7
|
+
| 1.x | :white_check_mark: |
|
|
8
|
+
| < 1.0 | :x: |
|
|
9
|
+
|
|
10
|
+
## Reporting a vulnerability
|
|
11
|
+
|
|
12
|
+
Please do not open a public issue for security problems. Instead, use GitHub's
|
|
13
|
+
private vulnerability reporting: go to the repository's **Security** tab and
|
|
14
|
+
choose **Report a vulnerability**, or visit
|
|
15
|
+
<https://github.com/dominicsayers/url_canonicalize/security/advisories/new>.
|
|
16
|
+
|
|
17
|
+
You should receive a response within a week. Please include a proof of concept
|
|
18
|
+
or reproduction steps where possible.
|
|
19
|
+
|
|
20
|
+
## Scope notes
|
|
21
|
+
|
|
22
|
+
URLCanonicalize fetches caller-supplied URLs, so requests forged through
|
|
23
|
+
redirects or canonical links (SSRF), TLS verification, resource exhaustion and
|
|
24
|
+
destination policy bypasses are all in scope. The README's "Security and
|
|
25
|
+
limits" section describes the protections the gem provides by default.
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
module Addressable
|
|
4
4
|
# Patch for Addressable's URI class
|
|
5
5
|
class URI
|
|
6
|
-
def self.canonicalize(uri)
|
|
6
|
+
def self.canonicalize(uri, **options)
|
|
7
7
|
url = parse(uri).to_s # uri can be anything Addressable::URI can handle
|
|
8
|
-
canonical_url = URLCanonicalize.canonicalize(url)
|
|
8
|
+
canonical_url = URLCanonicalize.canonicalize(url, **options)
|
|
9
9
|
parse(canonical_url)
|
|
10
10
|
end
|
|
11
11
|
end
|
data/lib/monkey_patches/uri.rb
CHANGED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module URLCanonicalize
|
|
4
|
+
# String-compatible response buffer that enforces a maximum byte size
|
|
5
|
+
class BoundedBody < String
|
|
6
|
+
def initialize(max_bytes)
|
|
7
|
+
@max_bytes = max_bytes
|
|
8
|
+
super()
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def <<(chunk)
|
|
12
|
+
raise URLCanonicalize::Exception::ResponseTooLarge, error_message if bytesize + chunk.bytesize > @max_bytes
|
|
13
|
+
|
|
14
|
+
super
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def error_message
|
|
20
|
+
"Response body exceeds #{@max_bytes} bytes"
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'ipaddr'
|
|
4
|
+
require 'socket'
|
|
5
|
+
|
|
6
|
+
module URLCanonicalize
|
|
7
|
+
# Resolves a URI to an address that is permitted by the fetch security policy
|
|
8
|
+
class Destination
|
|
9
|
+
FORBIDDEN_IPV4_RANGES = %w[
|
|
10
|
+
0.0.0.0/8
|
|
11
|
+
10.0.0.0/8
|
|
12
|
+
100.64.0.0/10
|
|
13
|
+
127.0.0.0/8
|
|
14
|
+
169.254.0.0/16
|
|
15
|
+
172.16.0.0/12
|
|
16
|
+
192.0.0.0/24
|
|
17
|
+
192.0.2.0/24
|
|
18
|
+
192.88.99.0/24
|
|
19
|
+
192.168.0.0/16
|
|
20
|
+
198.18.0.0/15
|
|
21
|
+
198.51.100.0/24
|
|
22
|
+
203.0.113.0/24
|
|
23
|
+
224.0.0.0/4
|
|
24
|
+
240.0.0.0/4
|
|
25
|
+
].map { |range| IPAddr.new(range) }.freeze
|
|
26
|
+
|
|
27
|
+
# Fail closed: only prefixes in IANA's IPv6 Global Unicast Address Space registry are accepted.
|
|
28
|
+
ALLOCATED_IPV6_RANGES = %w[
|
|
29
|
+
2001:200::/23
|
|
30
|
+
2001:400::/23
|
|
31
|
+
2001:600::/23
|
|
32
|
+
2001:800::/22
|
|
33
|
+
2001:c00::/23
|
|
34
|
+
2001:e00::/23
|
|
35
|
+
2001:1200::/23
|
|
36
|
+
2001:1400::/22
|
|
37
|
+
2001:1800::/23
|
|
38
|
+
2001:1a00::/23
|
|
39
|
+
2001:1c00::/22
|
|
40
|
+
2001:2000::/19
|
|
41
|
+
2001:4000::/23
|
|
42
|
+
2001:4200::/23
|
|
43
|
+
2001:4400::/23
|
|
44
|
+
2001:4600::/23
|
|
45
|
+
2001:4800::/23
|
|
46
|
+
2001:4a00::/23
|
|
47
|
+
2001:4c00::/23
|
|
48
|
+
2001:5000::/20
|
|
49
|
+
2001:8000::/19
|
|
50
|
+
2001:a000::/20
|
|
51
|
+
2001:b000::/20
|
|
52
|
+
2003::/18
|
|
53
|
+
2400::/12
|
|
54
|
+
2410::/12
|
|
55
|
+
2600::/12
|
|
56
|
+
2610::/23
|
|
57
|
+
2620::/23
|
|
58
|
+
2630::/12
|
|
59
|
+
2800::/12
|
|
60
|
+
2a00::/12
|
|
61
|
+
2a10::/12
|
|
62
|
+
2c00::/12
|
|
63
|
+
].map { |range| IPAddr.new(range) }.freeze
|
|
64
|
+
|
|
65
|
+
FORBIDDEN_IPV6_RANGES = %w[
|
|
66
|
+
2001::/23
|
|
67
|
+
2001:db8::/32
|
|
68
|
+
2002::/16
|
|
69
|
+
].map { |range| IPAddr.new(range) }.freeze
|
|
70
|
+
|
|
71
|
+
class << self
|
|
72
|
+
def resolve(uri, options)
|
|
73
|
+
validate!(uri, options)
|
|
74
|
+
addresses = resolve_addresses(uri)
|
|
75
|
+
validate_addresses!(uri, addresses, options)
|
|
76
|
+
addresses.first
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def validate!(uri, options)
|
|
80
|
+
raise URLCanonicalize::Exception::Security, 'URLs containing userinfo are not allowed' if uri.userinfo
|
|
81
|
+
return if options[:allowed_ports].include?(uri.port)
|
|
82
|
+
|
|
83
|
+
raise URLCanonicalize::Exception::Security, "Port #{uri.port} is not allowed"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def resolve_addresses(uri)
|
|
89
|
+
addresses = Addrinfo.getaddrinfo(uri.host, uri.port, nil, Socket::SOCK_STREAM)
|
|
90
|
+
addresses = addresses.map(&:ip_address).uniq
|
|
91
|
+
raise SocketError, "No addresses found for #{uri.host}" if addresses.empty?
|
|
92
|
+
|
|
93
|
+
addresses
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def validate_addresses!(uri, addresses, options)
|
|
97
|
+
return if options[:allow_private_networks]
|
|
98
|
+
|
|
99
|
+
addresses.each do |address|
|
|
100
|
+
next if public_address?(address)
|
|
101
|
+
|
|
102
|
+
raise URLCanonicalize::Exception::Security,
|
|
103
|
+
"Address #{address} for #{uri.host} is not publicly routable"
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def public_address?(value)
|
|
108
|
+
address = IPAddr.new(value)
|
|
109
|
+
address = address.native if address.ipv4_mapped?
|
|
110
|
+
|
|
111
|
+
if address.ipv4?
|
|
112
|
+
FORBIDDEN_IPV4_RANGES.none? { |range| range.include?(address) }
|
|
113
|
+
else
|
|
114
|
+
ALLOCATED_IPV6_RANGES.any? { |range| range.include?(address) } &&
|
|
115
|
+
FORBIDDEN_IPV6_RANGES.none? { |range| range.include?(address) }
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|