url_canonicalize 0.2.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/codeql-analysis.yml +70 -0
- data/.gitignore +2 -1
- data/.rubocop.yml +11 -10
- data/.ruby-gemset +0 -0
- data/.travis.yml +3 -5
- data/CHANGELOG.md +38 -2
- data/Gemfile +9 -6
- data/Gemfile.lock +110 -0
- data/README.md +0 -1
- data/circle.yml +29 -16
- data/lib/url_canonicalize/http.rb +13 -5
- data/lib/url_canonicalize/request.rb +6 -3
- data/lib/url_canonicalize/response.rb +2 -2
- data/lib/url_canonicalize/uri.rb +4 -1
- data/lib/url_canonicalize/version.rb +1 -1
- data/renovate.json +6 -0
- data/url_canonicalize.gemspec +11 -10
- metadata +11 -9
- data/Gemfile.local.example +0 -12
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 660347135be60bf3324f23f796121f9d82b7a8ab44924d2a03f8b1d4d5bff2cf
|
|
4
|
+
data.tar.gz: 0fd6bf4532334bbd631fe4b829cddac92d1780d679dbb77bff6b4a95b5475eb6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d9e654dcd8bf45a3261b5a7f9e2d1ce3f4e0f4dab80b48162cc8d9ae59493831060216b084b669820344414c4879557223faaabffc83f8003ca61f1a760c0d94
|
|
7
|
+
data.tar.gz: bb8f6459c22401e47fa050e367a9a3f7e719ac38f25758e6ecce4a30f6973c1b6e5753818127c215254cef8d9858356cd7b5c47ab55b882a1ba70b8971bcad82
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# For most projects, this workflow file will not need changing; you simply need
|
|
2
|
+
# to commit it to your repository.
|
|
3
|
+
#
|
|
4
|
+
# You may wish to alter this file to override the set of languages analyzed,
|
|
5
|
+
# or to provide custom queries or build logic.
|
|
6
|
+
#
|
|
7
|
+
# ******** NOTE ********
|
|
8
|
+
# We have attempted to detect the languages in your repository. Please check
|
|
9
|
+
# the `language` matrix defined below to confirm you have the correct set of
|
|
10
|
+
# supported CodeQL languages.
|
|
11
|
+
#
|
|
12
|
+
name: "CodeQL"
|
|
13
|
+
|
|
14
|
+
on:
|
|
15
|
+
push:
|
|
16
|
+
branches: [ main ]
|
|
17
|
+
pull_request:
|
|
18
|
+
# The branches below must be a subset of the branches above
|
|
19
|
+
branches: [ main ]
|
|
20
|
+
schedule:
|
|
21
|
+
- cron: '28 12 * * 5'
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
analyze:
|
|
25
|
+
name: Analyze
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
permissions:
|
|
28
|
+
actions: read
|
|
29
|
+
contents: read
|
|
30
|
+
security-events: write
|
|
31
|
+
|
|
32
|
+
strategy:
|
|
33
|
+
fail-fast: false
|
|
34
|
+
matrix:
|
|
35
|
+
language: [ 'ruby' ]
|
|
36
|
+
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
|
|
37
|
+
# Learn more about CodeQL language support at https://git.io/codeql-language-support
|
|
38
|
+
|
|
39
|
+
steps:
|
|
40
|
+
- name: Checkout repository
|
|
41
|
+
uses: actions/checkout@v4
|
|
42
|
+
|
|
43
|
+
# Initializes the CodeQL tools for scanning.
|
|
44
|
+
- name: Initialize CodeQL
|
|
45
|
+
uses: github/codeql-action/init@v3
|
|
46
|
+
with:
|
|
47
|
+
languages: ${{ matrix.language }}
|
|
48
|
+
# If you wish to specify custom queries, you can do so here or in a config file.
|
|
49
|
+
# By default, queries listed here will override any specified in a config file.
|
|
50
|
+
# Prefix the list here with "+" to use these queries and those in the config file.
|
|
51
|
+
# queries: ./path/to/local/query, your-org/your-repo/queries@main
|
|
52
|
+
|
|
53
|
+
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
|
54
|
+
# If this step fails, then you should remove it and run the build manually (see below)
|
|
55
|
+
- name: Autobuild
|
|
56
|
+
uses: github/codeql-action/autobuild@v3
|
|
57
|
+
|
|
58
|
+
# ℹ️ Command-line programs to run using the OS shell.
|
|
59
|
+
# 📚 https://git.io/JvXDl
|
|
60
|
+
|
|
61
|
+
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
|
62
|
+
# and modify them (or add more) to build your code if your project
|
|
63
|
+
# uses a compiled language
|
|
64
|
+
|
|
65
|
+
#- run: |
|
|
66
|
+
# make bootstrap
|
|
67
|
+
# make release
|
|
68
|
+
|
|
69
|
+
- name: Perform CodeQL Analysis
|
|
70
|
+
uses: github/codeql-action/analyze@v3
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -2,11 +2,8 @@
|
|
|
2
2
|
AllCops:
|
|
3
3
|
DisplayStyleGuide: true
|
|
4
4
|
DisplayCopNames: true
|
|
5
|
-
TargetRubyVersion:
|
|
6
|
-
|
|
7
|
-
- '*.gemspec'
|
|
8
|
-
- spec/**/*
|
|
9
|
-
|
|
5
|
+
TargetRubyVersion: 3.1
|
|
6
|
+
NewCops: enable
|
|
10
7
|
|
|
11
8
|
Layout/DotPosition:
|
|
12
9
|
Description: 'Checks the position of the dot in multi-line method calls.'
|
|
@@ -17,10 +14,19 @@ Layout/ExtraSpacing:
|
|
|
17
14
|
Description: 'Do not use unnecessary spacing.'
|
|
18
15
|
Enabled: true
|
|
19
16
|
|
|
17
|
+
Layout/LineLength:
|
|
18
|
+
Max: 120
|
|
19
|
+
Exclude:
|
|
20
|
+
- spec/**/*
|
|
21
|
+
|
|
20
22
|
Lint/LiteralInInterpolation:
|
|
21
23
|
Description: 'Avoid interpolating literals in strings'
|
|
22
24
|
AutoCorrect: true
|
|
23
25
|
|
|
26
|
+
Metrics/BlockLength:
|
|
27
|
+
Exclude:
|
|
28
|
+
- spec/**/*
|
|
29
|
+
|
|
24
30
|
Metrics/ClassLength:
|
|
25
31
|
CountComments: false # count full line comments?
|
|
26
32
|
Max: 200
|
|
@@ -28,11 +34,6 @@ Metrics/ClassLength:
|
|
|
28
34
|
Metrics/MethodLength:
|
|
29
35
|
Max: 12
|
|
30
36
|
|
|
31
|
-
Metrics/LineLength:
|
|
32
|
-
Max: 120
|
|
33
|
-
Exclude:
|
|
34
|
-
- spec/**/*
|
|
35
|
-
|
|
36
37
|
Naming/FileName:
|
|
37
38
|
Description: 'Use snake_case for source file names.'
|
|
38
39
|
Enabled: true
|
data/.ruby-gemset
CHANGED
|
File without changes
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,21 +1,39 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [Unreleased](https://github.com/dominicsayers/url_canonicalize/tree/HEAD)
|
|
4
|
+
|
|
5
|
+
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.15...HEAD)
|
|
6
|
+
|
|
7
|
+
**Implemented enhancements:**
|
|
8
|
+
|
|
9
|
+
- Feature: return a Nokogiri XML document if requested [\#5](https://github.com/dominicsayers/url_canonicalize/issues/5)
|
|
10
|
+
|
|
11
|
+
**Merged pull requests:**
|
|
12
|
+
|
|
13
|
+
- Version 0.2 [\#6](https://github.com/dominicsayers/url_canonicalize/pull/6) ([dominicsayers](https://github.com/dominicsayers))
|
|
2
14
|
|
|
3
15
|
## [v0.1.15](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.15) (2017-03-14)
|
|
16
|
+
|
|
4
17
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.14...v0.1.15)
|
|
5
18
|
|
|
6
19
|
## [v0.1.14](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.14) (2017-03-13)
|
|
20
|
+
|
|
7
21
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.13...v0.1.14)
|
|
8
22
|
|
|
9
23
|
## [v0.1.13](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.13) (2017-03-13)
|
|
24
|
+
|
|
10
25
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.12...v0.1.13)
|
|
11
26
|
|
|
12
27
|
## [v0.1.12](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.12) (2017-03-13)
|
|
28
|
+
|
|
13
29
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.11...v0.1.12)
|
|
14
30
|
|
|
15
31
|
## [v0.1.11](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.11) (2017-03-13)
|
|
32
|
+
|
|
16
33
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.10...v0.1.11)
|
|
17
34
|
|
|
18
35
|
## [v0.1.10](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.10) (2017-03-13)
|
|
36
|
+
|
|
19
37
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.9...v0.1.10)
|
|
20
38
|
|
|
21
39
|
**Closed issues:**
|
|
@@ -23,9 +41,11 @@
|
|
|
23
41
|
- URLCanonicalize doesn't accept host names [\#4](https://github.com/dominicsayers/url_canonicalize/issues/4)
|
|
24
42
|
|
|
25
43
|
## [v0.1.9](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.9) (2017-03-06)
|
|
44
|
+
|
|
26
45
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.8...v0.1.9)
|
|
27
46
|
|
|
28
47
|
## [v0.1.8](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.8) (2017-03-06)
|
|
48
|
+
|
|
29
49
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.7...v0.1.8)
|
|
30
50
|
|
|
31
51
|
**Closed issues:**
|
|
@@ -33,6 +53,7 @@
|
|
|
33
53
|
- Use a later version of Ruby on CircleCI [\#3](https://github.com/dominicsayers/url_canonicalize/issues/3)
|
|
34
54
|
|
|
35
55
|
## [v0.1.7](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.7) (2017-03-06)
|
|
56
|
+
|
|
36
57
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.6...v0.1.7)
|
|
37
58
|
|
|
38
59
|
**Closed issues:**
|
|
@@ -40,42 +61,57 @@
|
|
|
40
61
|
- Insecure version of Nokogiri [\#2](https://github.com/dominicsayers/url_canonicalize/issues/2)
|
|
41
62
|
|
|
42
63
|
## [v0.1.6](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.6) (2017-03-06)
|
|
64
|
+
|
|
43
65
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.5...v0.1.6)
|
|
44
66
|
|
|
45
67
|
## [v0.1.5](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.5) (2016-10-26)
|
|
68
|
+
|
|
46
69
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.4...v0.1.5)
|
|
47
70
|
|
|
48
71
|
## [v0.1.4](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.4) (2016-10-26)
|
|
72
|
+
|
|
49
73
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.2...v0.1.4)
|
|
50
74
|
|
|
51
75
|
## [v0.1.2](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.2) (2016-10-26)
|
|
76
|
+
|
|
52
77
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.1...v0.1.2)
|
|
53
78
|
|
|
54
79
|
## [v0.1.1](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.1) (2016-10-26)
|
|
80
|
+
|
|
55
81
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.1.0...v0.1.1)
|
|
56
82
|
|
|
57
83
|
## [v0.1.0](https://github.com/dominicsayers/url_canonicalize/tree/v0.1.0) (2016-10-20)
|
|
84
|
+
|
|
58
85
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.7...v0.1.0)
|
|
59
86
|
|
|
60
87
|
## [v0.0.7](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.7) (2016-10-20)
|
|
88
|
+
|
|
61
89
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.6...v0.0.7)
|
|
62
90
|
|
|
63
91
|
## [v0.0.6](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.6) (2016-10-20)
|
|
92
|
+
|
|
64
93
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.5...v0.0.6)
|
|
65
94
|
|
|
66
95
|
## [v0.0.5](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.5) (2016-10-20)
|
|
96
|
+
|
|
67
97
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.4...v0.0.5)
|
|
68
98
|
|
|
69
99
|
## [v0.0.4](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.4) (2016-10-20)
|
|
100
|
+
|
|
70
101
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.3...v0.0.4)
|
|
71
102
|
|
|
72
103
|
## [v0.0.3](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.3) (2016-10-20)
|
|
104
|
+
|
|
73
105
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.2...v0.0.3)
|
|
74
106
|
|
|
75
107
|
## [v0.0.2](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.2) (2016-10-20)
|
|
108
|
+
|
|
76
109
|
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/v0.0.1...v0.0.2)
|
|
77
110
|
|
|
78
111
|
## [v0.0.1](https://github.com/dominicsayers/url_canonicalize/tree/v0.0.1) (2016-10-20)
|
|
79
112
|
|
|
113
|
+
[Full Changelog](https://github.com/dominicsayers/url_canonicalize/compare/fec855bec42813304f93c8ef87035f184b054344...v0.0.1)
|
|
114
|
+
|
|
115
|
+
|
|
80
116
|
|
|
81
|
-
\* *This
|
|
117
|
+
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
|
data/Gemfile
CHANGED
|
@@ -4,16 +4,19 @@ source 'https://rubygems.org'
|
|
|
4
4
|
|
|
5
5
|
gemspec
|
|
6
6
|
|
|
7
|
+
group :static_code_analysis do
|
|
8
|
+
gem 'rubocop', require: false
|
|
9
|
+
gem 'rubocop-rspec', require: false
|
|
10
|
+
end
|
|
11
|
+
|
|
7
12
|
group :test do
|
|
8
|
-
gem 'coveralls', require: false
|
|
13
|
+
gem 'coveralls-ruby', require: false
|
|
9
14
|
gem 'rspec'
|
|
10
15
|
gem 'rspec_junit_formatter'
|
|
11
|
-
gem 'simplecov'
|
|
16
|
+
gem 'simplecov'
|
|
12
17
|
gem 'webmock'
|
|
13
18
|
end
|
|
14
19
|
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
if File.exist?(local_gemfile)
|
|
18
|
-
eval(File.read(local_gemfile)) # rubocop:disable Security/Eval
|
|
20
|
+
group :build do
|
|
21
|
+
gem 'gem-release', require: false
|
|
19
22
|
end
|
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
url_canonicalize (1.0.0)
|
|
5
|
+
addressable (~> 2)
|
|
6
|
+
nokogiri (>= 1.13)
|
|
7
|
+
|
|
8
|
+
GEM
|
|
9
|
+
remote: https://rubygems.org/
|
|
10
|
+
specs:
|
|
11
|
+
addressable (2.8.6)
|
|
12
|
+
public_suffix (>= 2.0.2, < 6.0)
|
|
13
|
+
ast (2.4.2)
|
|
14
|
+
coveralls-ruby (0.2.0)
|
|
15
|
+
json (>= 1.8, < 3)
|
|
16
|
+
simplecov (~> 0.16.1)
|
|
17
|
+
term-ansicolor (~> 1.3)
|
|
18
|
+
thor (>= 0.19.4, < 2.0)
|
|
19
|
+
tins (~> 1.6)
|
|
20
|
+
crack (0.4.5)
|
|
21
|
+
rexml
|
|
22
|
+
diff-lcs (1.5.0)
|
|
23
|
+
docile (1.4.0)
|
|
24
|
+
gem-release (2.2.2)
|
|
25
|
+
hashdiff (1.1.0)
|
|
26
|
+
json (2.7.1)
|
|
27
|
+
language_server-protocol (3.17.0.3)
|
|
28
|
+
mini_portile2 (2.8.5)
|
|
29
|
+
nokogiri (1.13.10)
|
|
30
|
+
mini_portile2 (~> 2.8.0)
|
|
31
|
+
racc (~> 1.4)
|
|
32
|
+
parallel (1.24.0)
|
|
33
|
+
parser (3.3.0.5)
|
|
34
|
+
ast (~> 2.4.1)
|
|
35
|
+
racc
|
|
36
|
+
public_suffix (4.0.7)
|
|
37
|
+
racc (1.6.1)
|
|
38
|
+
rainbow (3.1.1)
|
|
39
|
+
regexp_parser (2.9.0)
|
|
40
|
+
rexml (3.2.6)
|
|
41
|
+
rspec (3.12.0)
|
|
42
|
+
rspec-core (~> 3.12.0)
|
|
43
|
+
rspec-expectations (~> 3.12.0)
|
|
44
|
+
rspec-mocks (~> 3.12.0)
|
|
45
|
+
rspec-core (3.12.2)
|
|
46
|
+
rspec-support (~> 3.12.0)
|
|
47
|
+
rspec-expectations (3.12.3)
|
|
48
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
49
|
+
rspec-support (~> 3.12.0)
|
|
50
|
+
rspec-mocks (3.12.6)
|
|
51
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
52
|
+
rspec-support (~> 3.12.0)
|
|
53
|
+
rspec-support (3.12.1)
|
|
54
|
+
rspec_junit_formatter (0.6.0)
|
|
55
|
+
rspec-core (>= 2, < 4, != 2.12.0)
|
|
56
|
+
rubocop (1.60.2)
|
|
57
|
+
json (~> 2.3)
|
|
58
|
+
language_server-protocol (>= 3.17.0)
|
|
59
|
+
parallel (~> 1.10)
|
|
60
|
+
parser (>= 3.3.0.2)
|
|
61
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
62
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
63
|
+
rexml (>= 3.2.5, < 4.0)
|
|
64
|
+
rubocop-ast (>= 1.30.0, < 2.0)
|
|
65
|
+
ruby-progressbar (~> 1.7)
|
|
66
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
|
67
|
+
rubocop-ast (1.30.0)
|
|
68
|
+
parser (>= 3.2.1.0)
|
|
69
|
+
rubocop-capybara (2.20.0)
|
|
70
|
+
rubocop (~> 1.41)
|
|
71
|
+
rubocop-factory_bot (2.25.1)
|
|
72
|
+
rubocop (~> 1.41)
|
|
73
|
+
rubocop-rspec (2.26.1)
|
|
74
|
+
rubocop (~> 1.40)
|
|
75
|
+
rubocop-capybara (~> 2.17)
|
|
76
|
+
rubocop-factory_bot (~> 2.22)
|
|
77
|
+
ruby-progressbar (1.13.0)
|
|
78
|
+
simplecov (0.16.1)
|
|
79
|
+
docile (~> 1.1)
|
|
80
|
+
json (>= 1.8, < 3)
|
|
81
|
+
simplecov-html (~> 0.10.0)
|
|
82
|
+
simplecov-html (0.10.2)
|
|
83
|
+
sync (0.5.0)
|
|
84
|
+
term-ansicolor (1.7.1)
|
|
85
|
+
tins (~> 1.0)
|
|
86
|
+
thor (1.2.1)
|
|
87
|
+
tins (1.31.0)
|
|
88
|
+
sync
|
|
89
|
+
unicode-display_width (2.5.0)
|
|
90
|
+
webmock (3.19.1)
|
|
91
|
+
addressable (>= 2.8.0)
|
|
92
|
+
crack (>= 0.3.2)
|
|
93
|
+
hashdiff (>= 0.4.0, < 2.0.0)
|
|
94
|
+
|
|
95
|
+
PLATFORMS
|
|
96
|
+
x86_64-linux
|
|
97
|
+
|
|
98
|
+
DEPENDENCIES
|
|
99
|
+
coveralls-ruby
|
|
100
|
+
gem-release
|
|
101
|
+
rspec
|
|
102
|
+
rspec_junit_formatter
|
|
103
|
+
rubocop
|
|
104
|
+
rubocop-rspec
|
|
105
|
+
simplecov
|
|
106
|
+
url_canonicalize!
|
|
107
|
+
webmock
|
|
108
|
+
|
|
109
|
+
BUNDLED WITH
|
|
110
|
+
2.5.5
|
data/README.md
CHANGED
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
[](https://rubygems.org/gems/url_canonicalize)
|
|
4
4
|
[](https://circleci.com/gh/dominicsayers/url_canonicalize)
|
|
5
5
|
[](https://codeclimate.com/github/dominicsayers/url_canonicalize/maintainability)
|
|
6
|
-
[](https://codeclimate.com/github/dominicsayers/url_canonicalize/test_coverage)
|
|
7
6
|
[](https://coveralls.io/github/dominicsayers/url_canonicalize?branch=master)
|
|
8
7
|
[](https://dependencyci.com/github/dominicsayers/url_canonicalize)
|
|
9
8
|
[](https://hakiri.io/github/dominicsayers/url_canonicalize/master)
|
data/circle.yml
CHANGED
|
@@ -1,16 +1,29 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
1
|
+
version: 2.1
|
|
2
|
+
orbs:
|
|
3
|
+
ruby: circleci/ruby@1.1
|
|
4
|
+
jobs:
|
|
5
|
+
check_and_test:
|
|
6
|
+
parameters:
|
|
7
|
+
ruby-version:
|
|
8
|
+
type: string
|
|
9
|
+
docker:
|
|
10
|
+
- image: cimg/ruby:<< parameters.ruby-version >>
|
|
11
|
+
parallelism: 3
|
|
12
|
+
steps:
|
|
13
|
+
- checkout
|
|
14
|
+
- ruby/install-deps
|
|
15
|
+
- run:
|
|
16
|
+
name: Run static code analysis
|
|
17
|
+
command: bundle exec rubocop
|
|
18
|
+
- run:
|
|
19
|
+
name: Run tests
|
|
20
|
+
command: bundle exec rspec
|
|
21
|
+
workflows:
|
|
22
|
+
version: 2
|
|
23
|
+
test:
|
|
24
|
+
jobs:
|
|
25
|
+
- check_and_test:
|
|
26
|
+
# cribbed from http://mikebian.co/running-tests-against-multiple-ruby-versions-using-circleci/
|
|
27
|
+
matrix:
|
|
28
|
+
parameters:
|
|
29
|
+
ruby-version: ["3.1", "3.2", "3.3"]
|
|
@@ -72,6 +72,7 @@ module URLCanonicalize
|
|
|
72
72
|
def redirect_loop_detected?
|
|
73
73
|
if redirect_list.include?(response_url)
|
|
74
74
|
return true if last_known_good
|
|
75
|
+
|
|
75
76
|
raise URLCanonicalize::Exception::Redirect, 'Redirect loop detected'
|
|
76
77
|
end
|
|
77
78
|
|
|
@@ -84,6 +85,7 @@ module URLCanonicalize
|
|
|
84
85
|
def max_redirects_reached?
|
|
85
86
|
return false unless @redirects > options[:max_redirects]
|
|
86
87
|
return true if last_known_good
|
|
88
|
+
|
|
87
89
|
raise URLCanonicalize::Exception::Redirect, "#{@redirects} redirects is too many"
|
|
88
90
|
end
|
|
89
91
|
|
|
@@ -102,6 +104,7 @@ module URLCanonicalize
|
|
|
102
104
|
def handle_canonical_found
|
|
103
105
|
self.last_known_good = response.response
|
|
104
106
|
return true if response_url == url || redirect_list.include?(response_url)
|
|
107
|
+
|
|
105
108
|
set_url_from_response
|
|
106
109
|
false
|
|
107
110
|
end
|
|
@@ -112,6 +115,7 @@ module URLCanonicalize
|
|
|
112
115
|
|
|
113
116
|
def handle_failure
|
|
114
117
|
return true if last_known_good
|
|
118
|
+
|
|
115
119
|
raise URLCanonicalize::Exception::Failure, "#{response.failure_class}: #{response.message}"
|
|
116
120
|
end
|
|
117
121
|
|
|
@@ -149,14 +153,18 @@ module URLCanonicalize
|
|
|
149
153
|
h.open_timeout = options[:open_timeout]
|
|
150
154
|
h.read_timeout = options[:read_timeout]
|
|
151
155
|
|
|
156
|
+
ssl!(h)
|
|
157
|
+
|
|
158
|
+
h
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def ssl!(http)
|
|
152
162
|
if uri.scheme == 'https'
|
|
153
|
-
|
|
154
|
-
|
|
163
|
+
http.use_ssl = true # Can generate exception
|
|
164
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
155
165
|
else
|
|
156
|
-
|
|
166
|
+
http.use_ssl = false
|
|
157
167
|
end
|
|
158
|
-
|
|
159
|
-
h
|
|
160
168
|
end
|
|
161
169
|
|
|
162
170
|
def options
|
|
@@ -38,7 +38,7 @@ module URLCanonicalize
|
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
# We can stub this method in testing then call #response any number of times
|
|
41
|
-
def do_http_request
|
|
41
|
+
def do_http_request # :nodoc: internal use only
|
|
42
42
|
http.do_request request # Some URLs can throw an exception here
|
|
43
43
|
end
|
|
44
44
|
|
|
@@ -147,8 +147,8 @@ module URLCanonicalize
|
|
|
147
147
|
def headers
|
|
148
148
|
@headers ||= {
|
|
149
149
|
'Accept-Language' => 'en-US,en;q=0.8',
|
|
150
|
-
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) '\
|
|
151
|
-
'AppleWebKit/537.36 (KHTML, like Gecko) '\
|
|
150
|
+
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) ' \
|
|
151
|
+
'AppleWebKit/537.36 (KHTML, like Gecko) ' \
|
|
152
152
|
'Chrome/51.0.2704.103 Safari/537.36'
|
|
153
153
|
}
|
|
154
154
|
end
|
|
@@ -170,6 +170,7 @@ module URLCanonicalize
|
|
|
170
170
|
|
|
171
171
|
def relative_to_absolute(partial_url)
|
|
172
172
|
return unless partial_url
|
|
173
|
+
|
|
173
174
|
partial_uri = ::URI.parse(partial_url)
|
|
174
175
|
|
|
175
176
|
if partial_uri.host
|
|
@@ -183,9 +184,11 @@ module URLCanonicalize
|
|
|
183
184
|
|
|
184
185
|
def log_response
|
|
185
186
|
return unless ENV['DEBUG']
|
|
187
|
+
|
|
186
188
|
puts "#{http_method.upcase} #{url} #{response.code} #{response.message}"
|
|
187
189
|
|
|
188
190
|
return unless ENV['DEBUG'].casecmp('headers')
|
|
191
|
+
|
|
189
192
|
response.each { |k, v| puts " #{k}:\t#{v}" }
|
|
190
193
|
end
|
|
191
194
|
|
|
@@ -28,7 +28,7 @@ module URLCanonicalize
|
|
|
28
28
|
def initialize(url, response, html)
|
|
29
29
|
@response = response
|
|
30
30
|
@html = html
|
|
31
|
-
super
|
|
31
|
+
super(url)
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
@@ -40,7 +40,7 @@ module URLCanonicalize
|
|
|
40
40
|
|
|
41
41
|
def initialize(url, response)
|
|
42
42
|
@response = response
|
|
43
|
-
super
|
|
43
|
+
super(url)
|
|
44
44
|
end
|
|
45
45
|
end
|
|
46
46
|
|
data/lib/url_canonicalize/uri.rb
CHANGED
|
@@ -18,16 +18,19 @@ module URLCanonicalize
|
|
|
18
18
|
def valid?(uri)
|
|
19
19
|
raise URLCanonicalize::Exception::URI, "#{uri} must be http or https" unless VALID_CLASSES.include?(uri.class)
|
|
20
20
|
raise URLCanonicalize::Exception::URI, "Missing host name in #{uri}" unless uri.host
|
|
21
|
+
raise URLCanonicalize::Exception::URI, "Empty host name in #{uri}" if uri.host.empty?
|
|
22
|
+
|
|
21
23
|
true
|
|
22
24
|
end
|
|
23
25
|
|
|
24
26
|
def decorate(url)
|
|
25
27
|
return url if url.include? COLON
|
|
28
|
+
|
|
26
29
|
"http://#{url}" # Add protocol if we just receive a host name
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
VALID_CLASSES = [::URI::HTTP, ::URI::HTTPS].freeze
|
|
30
|
-
COLON = ':'
|
|
33
|
+
COLON = ':'
|
|
31
34
|
end
|
|
32
35
|
end
|
|
33
36
|
end
|
data/renovate.json
ADDED
data/url_canonicalize.gemspec
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
require 'url_canonicalize/version'
|
|
5
6
|
|
|
@@ -9,20 +10,20 @@ Gem::Specification.new do |s|
|
|
|
9
10
|
s.authors = ['Dominic Sayers']
|
|
10
11
|
s.email = ['dominic@sayers.cc']
|
|
11
12
|
s.summary = 'Finds the canonical version of a URL'
|
|
12
|
-
s.description = 'Rubygem that finds the canonical version of a URL by '\
|
|
13
|
-
'providing #canonicalize methods for the String, URI::HTTP'\
|
|
13
|
+
s.description = 'Rubygem that finds the canonical version of a URL by ' \
|
|
14
|
+
'providing #canonicalize methods for the String, URI::HTTP' \
|
|
14
15
|
', URI::HTTPS and Addressable::URI classes'
|
|
15
16
|
s.homepage = 'https://github.com/dominicsayers/url_canonicalize'
|
|
16
17
|
s.license = 'MIT'
|
|
17
18
|
|
|
18
|
-
s.
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
s.required_ruby_version = '>= 3.1.0'
|
|
20
|
+
|
|
21
|
+
s.files = `git ls-files`.split($RS).grep_v(%r{^spec/})
|
|
21
22
|
|
|
22
|
-
s.test_files = []
|
|
23
23
|
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
24
24
|
s.require_paths = ['lib']
|
|
25
25
|
|
|
26
|
-
s.
|
|
27
|
-
s.
|
|
26
|
+
s.add_runtime_dependency 'addressable', '~> 2' # To normalize URLs
|
|
27
|
+
s.add_runtime_dependency 'nokogiri', '>= 1.13' # To look for <link rel="canonical" ...> in HTML
|
|
28
|
+
s.metadata['rubygems_mfa_required'] = 'true'
|
|
28
29
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: url_canonicalize
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Dominic Sayers
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2024-01-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: addressable
|
|
@@ -30,14 +30,14 @@ dependencies:
|
|
|
30
30
|
requirements:
|
|
31
31
|
- - ">="
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: 1.
|
|
33
|
+
version: '1.13'
|
|
34
34
|
type: :runtime
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: 1.
|
|
40
|
+
version: '1.13'
|
|
41
41
|
description: 'Rubygem that finds the canonical version of a URL by providing #canonicalize
|
|
42
42
|
methods for the String, URI::HTTP, URI::HTTPS and Addressable::URI classes'
|
|
43
43
|
email:
|
|
@@ -47,6 +47,7 @@ extensions: []
|
|
|
47
47
|
extra_rdoc_files: []
|
|
48
48
|
files:
|
|
49
49
|
- ".codeclimate.yml"
|
|
50
|
+
- ".github/workflows/codeql-analysis.yml"
|
|
50
51
|
- ".gitignore"
|
|
51
52
|
- ".hound.yml"
|
|
52
53
|
- ".rspec"
|
|
@@ -55,7 +56,7 @@ files:
|
|
|
55
56
|
- ".travis.yml"
|
|
56
57
|
- CHANGELOG.md
|
|
57
58
|
- Gemfile
|
|
58
|
-
- Gemfile.
|
|
59
|
+
- Gemfile.lock
|
|
59
60
|
- Guardfile
|
|
60
61
|
- LICENSE
|
|
61
62
|
- README.md
|
|
@@ -71,11 +72,13 @@ files:
|
|
|
71
72
|
- lib/url_canonicalize/response.rb
|
|
72
73
|
- lib/url_canonicalize/uri.rb
|
|
73
74
|
- lib/url_canonicalize/version.rb
|
|
75
|
+
- renovate.json
|
|
74
76
|
- url_canonicalize.gemspec
|
|
75
77
|
homepage: https://github.com/dominicsayers/url_canonicalize
|
|
76
78
|
licenses:
|
|
77
79
|
- MIT
|
|
78
|
-
metadata:
|
|
80
|
+
metadata:
|
|
81
|
+
rubygems_mfa_required: 'true'
|
|
79
82
|
post_install_message:
|
|
80
83
|
rdoc_options: []
|
|
81
84
|
require_paths:
|
|
@@ -84,15 +87,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
84
87
|
requirements:
|
|
85
88
|
- - ">="
|
|
86
89
|
- !ruby/object:Gem::Version
|
|
87
|
-
version:
|
|
90
|
+
version: 3.1.0
|
|
88
91
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
92
|
requirements:
|
|
90
93
|
- - ">="
|
|
91
94
|
- !ruby/object:Gem::Version
|
|
92
95
|
version: '0'
|
|
93
96
|
requirements: []
|
|
94
|
-
|
|
95
|
-
rubygems_version: 2.7.7
|
|
97
|
+
rubygems_version: 3.4.6
|
|
96
98
|
signing_key:
|
|
97
99
|
specification_version: 4
|
|
98
100
|
summary: Finds the canonical version of a URL
|
data/Gemfile.local.example
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
# Copy this file to a file named Gemfile.local to add these gems to your dev toolset
|
|
2
|
-
# Feel free to modify Gemfile.local to suit your own preferences
|
|
3
|
-
group :development do
|
|
4
|
-
gem 'github_changelog_generator'
|
|
5
|
-
gem 'gem-release'
|
|
6
|
-
gem 'guard'
|
|
7
|
-
gem 'guard-rspec'
|
|
8
|
-
gem 'guard-rubocop'
|
|
9
|
-
gem 'listen', '~> 3.0', '< 3.1' # Dependency of guard, 3.1 requires Ruby 2.2+
|
|
10
|
-
gem 'rake'
|
|
11
|
-
gem 'rubocop'
|
|
12
|
-
end
|