ruby-link-checker 0.3.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/coverage.yml +24 -0
- data/.github/workflows/danger-comment.yml +17 -0
- data/.github/workflows/danger.yml +12 -0
- data/.github/workflows/lint.yml +20 -0
- data/.github/workflows/release.yml +14 -0
- data/.github/workflows/test.yml +25 -0
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/.rubocop.yml +26 -0
- data/.rubocop_todo.yml +332 -0
- data/AGENTS.md +23 -0
- data/CHANGELOG.md +27 -0
- data/CONTRIBUTING.md +4 -0
- data/Dangerfile +2 -0
- data/Gemfile +5 -1
- data/README.md +151 -7
- data/RELEASING.md +17 -40
- data/Rakefile +8 -0
- data/benchmarks/check.rb +27 -20
- data/lib/ruby-link-checker/async/http/checker.rb +118 -0
- data/lib/ruby-link-checker/async/http/config.rb +39 -0
- data/lib/ruby-link-checker/async/http/result.rb +85 -0
- data/lib/ruby-link-checker/async/http.rb +3 -0
- data/lib/ruby-link-checker/cache.rb +101 -0
- data/lib/ruby-link-checker/callbacks.rb +2 -2
- data/lib/ruby-link-checker/checker.rb +76 -4
- data/lib/ruby-link-checker/config.rb +64 -0
- data/lib/ruby-link-checker/errors/connection_error.rb +23 -0
- data/lib/ruby-link-checker/errors/redirect_limit_error.rb +15 -0
- data/lib/ruby-link-checker/errors.rb +2 -0
- data/lib/ruby-link-checker/net/http/checker.rb +4 -1
- data/lib/ruby-link-checker/net/http/result.rb +4 -0
- data/lib/ruby-link-checker/result.rb +25 -0
- data/lib/ruby-link-checker/task.rb +2 -0
- data/lib/ruby-link-checker/tasks.rb +43 -7
- data/lib/ruby-link-checker/typhoeus/hydra/checker.rb +29 -6
- data/lib/ruby-link-checker/typhoeus/hydra/result.rb +4 -0
- data/lib/ruby-link-checker/version.rb +1 -1
- data/lib/ruby-link-checker.rb +2 -0
- data/ruby-link-checker.gemspec +3 -2
- data/spec/ruby-link-checker/async/http/checker_spec.rb +184 -0
- data/spec/ruby-link-checker/callbacks_spec.rb +33 -0
- data/spec/ruby-link-checker/checker_spec.rb +89 -2
- data/spec/ruby-link-checker/config_spec.rb +6 -0
- data/spec/ruby-link-checker/errors/redirect_loop_error_spec.rb +18 -0
- data/spec/ruby-link-checker/net/http/checker_spec.rb +32 -0
- data/spec/ruby-link-checker/result_error_spec.rb +34 -0
- data/spec/ruby-link-checker/result_spec.rb +41 -0
- data/spec/ruby-link-checker/task_spec.rb +20 -0
- data/spec/ruby-link-checker/tasks_spec.rb +91 -0
- data/spec/ruby-link-checker/typhoeus/hydra/checker_spec.rb +116 -3
- data/spec/spec_helper.rb +3 -1
- data/spec/support/link_checker.rb +267 -3
- metadata +42 -35
- data/Gemfile.lock +0 -215
- data/coverage/assets/0.13.1/DataTables-1.10.20/images/sort_asc.png +0 -0
- data/coverage/assets/0.13.1/DataTables-1.10.20/images/sort_asc_disabled.png +0 -0
- data/coverage/assets/0.13.1/DataTables-1.10.20/images/sort_both.png +0 -0
- data/coverage/assets/0.13.1/DataTables-1.10.20/images/sort_desc.png +0 -0
- data/coverage/assets/0.13.1/DataTables-1.10.20/images/sort_desc_disabled.png +0 -0
- data/coverage/assets/0.13.1/application.css +0 -1
- data/coverage/assets/0.13.1/application.js +0 -7
- data/coverage/assets/0.13.1/colorbox/border.png +0 -0
- data/coverage/assets/0.13.1/colorbox/controls.png +0 -0
- data/coverage/assets/0.13.1/colorbox/loading.gif +0 -0
- data/coverage/assets/0.13.1/colorbox/loading_background.png +0 -0
- data/coverage/assets/0.13.1/favicon_green.png +0 -0
- data/coverage/assets/0.13.1/favicon_red.png +0 -0
- data/coverage/assets/0.13.1/favicon_yellow.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_flat_0_aaaaaa_40x100.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_flat_75_ffffff_40x100.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_glass_55_fbf9ee_1x400.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_glass_65_ffffff_1x400.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_glass_75_dadada_1x400.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_glass_95_fef1ec_1x400.png +0 -0
- data/coverage/assets/0.13.1/images/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
- data/coverage/assets/0.13.1/images/ui-icons_222222_256x240.png +0 -0
- data/coverage/assets/0.13.1/images/ui-icons_2e83ff_256x240.png +0 -0
- data/coverage/assets/0.13.1/images/ui-icons_454545_256x240.png +0 -0
- data/coverage/assets/0.13.1/images/ui-icons_888888_256x240.png +0 -0
- data/coverage/assets/0.13.1/images/ui-icons_cd0a0a_256x240.png +0 -0
- data/coverage/assets/0.13.1/loading.gif +0 -0
- data/coverage/assets/0.13.1/magnify.png +0 -0
- data/coverage/index.html +0 -15014
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'spec_helper'
|
|
4
4
|
|
|
5
5
|
describe LinkChecker::Typhoeus::Hydra::Checker do
|
|
6
|
-
module
|
|
6
|
+
module TyphoeusTestLinkChecker
|
|
7
7
|
class Task < LinkChecker::Typhoeus::Hydra::Task; end
|
|
8
8
|
|
|
9
9
|
class LinkChecker < LinkChecker::Typhoeus::Hydra::Checker
|
|
@@ -15,13 +15,19 @@ describe LinkChecker::Typhoeus::Hydra::Checker do
|
|
|
15
15
|
end
|
|
16
16
|
before :all do
|
|
17
17
|
VCR.configure do |config|
|
|
18
|
-
config.hook_into :
|
|
18
|
+
config.hook_into :webmock
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
describe
|
|
22
|
+
describe TyphoeusTestLinkChecker::LinkChecker do
|
|
23
23
|
it_behaves_like 'a link checker'
|
|
24
24
|
|
|
25
|
+
describe '.config' do
|
|
26
|
+
it 'returns the Config module' do
|
|
27
|
+
expect(LinkChecker::Typhoeus::Hydra.config).to eq LinkChecker::Typhoeus::Hydra::Config
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
25
31
|
context 'with timeout options', vcr: { cassette_name: '200' } do
|
|
26
32
|
before do
|
|
27
33
|
LinkChecker::Typhoeus::Hydra.configure do |config|
|
|
@@ -64,4 +70,111 @@ describe LinkChecker::Typhoeus::Hydra::Checker do
|
|
|
64
70
|
end
|
|
65
71
|
end
|
|
66
72
|
end
|
|
73
|
+
|
|
74
|
+
describe LinkChecker::Typhoeus::Hydra::Result do
|
|
75
|
+
describe '#code' do
|
|
76
|
+
it 'returns -1 when the response raises' do
|
|
77
|
+
request = instance_double(Typhoeus::Request)
|
|
78
|
+
response = instance_double(Typhoeus::Response)
|
|
79
|
+
allow(response).to receive(:code).and_raise(StandardError)
|
|
80
|
+
result = described_class.new(URI('https://www.example.org'), 'GET', nil, request, response, {})
|
|
81
|
+
expect(result.code).to eq(-1)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
describe '#run' do
|
|
87
|
+
it 'delegates to the hydra' do
|
|
88
|
+
checker = described_class.new
|
|
89
|
+
expect(checker.instance_variable_get(:@hydra)).to receive(:run)
|
|
90
|
+
checker.run
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
describe 'delayed retries' do
|
|
95
|
+
let(:checker) { described_class.new(methods: ['GET'], retries: 1, retry_after: 0.005) }
|
|
96
|
+
|
|
97
|
+
around do |example|
|
|
98
|
+
VCR.turned_off { example.run }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
it 'processes unrelated requests before waiting for a retry' do
|
|
102
|
+
completed = []
|
|
103
|
+
stub_request(:get, 'https://www.example.org/retry').to_return({ status: 503 }, { status: 200 })
|
|
104
|
+
stub_request(:get, 'https://www.example.org/other').to_return(status: 200)
|
|
105
|
+
checker.on(:success) { |result| completed << result.uri.path }
|
|
106
|
+
checker.check('https://www.example.org/retry')
|
|
107
|
+
checker.check('https://www.example.org/other')
|
|
108
|
+
checker.run
|
|
109
|
+
expect(completed).to eq(['/other', '/retry'])
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
describe 'transport failures' do
|
|
114
|
+
let(:checker) { described_class.new(methods: ['GET']) }
|
|
115
|
+
|
|
116
|
+
around do |example|
|
|
117
|
+
VCR.turned_off do
|
|
118
|
+
WebMock.disable!
|
|
119
|
+
Typhoeus::Config.block_connection = true
|
|
120
|
+
example.run
|
|
121
|
+
ensure
|
|
122
|
+
Typhoeus::Config.block_connection = false
|
|
123
|
+
WebMock.enable!
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
after do
|
|
128
|
+
Typhoeus::Expectation.clear
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
it 'reports connection errors separately from HTTP failures' do
|
|
132
|
+
response = Typhoeus::Response.new(code: 0, return_code: :couldnt_connect)
|
|
133
|
+
Typhoeus.stub(nil).and_return(response)
|
|
134
|
+
checker.check('https://www.example.org/')
|
|
135
|
+
checker.run
|
|
136
|
+
expect(checker.results[:failure]).to be_empty
|
|
137
|
+
expect(checker.results[:error].first.error_type).to eq(:connection)
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
describe 'in-flight deduplication' do
|
|
142
|
+
let(:checker) { described_class.new(methods: ['GET'], deduplicate: true) }
|
|
143
|
+
let(:url) { 'https://www.example.org/' }
|
|
144
|
+
|
|
145
|
+
around do |example|
|
|
146
|
+
VCR.turned_off { example.run }
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
it 'fans out events to duplicates queued before completion' do
|
|
150
|
+
metadata = []
|
|
151
|
+
stub_request(:any, url).to_return(status: 200)
|
|
152
|
+
checker.on(:result) { |result| metadata << result.options }
|
|
153
|
+
checker.check(url, location: 'one.html')
|
|
154
|
+
checker.check(url, location: 'two.html')
|
|
155
|
+
checker.run
|
|
156
|
+
expect(a_request(:any, url)).to have_been_made.once
|
|
157
|
+
expect(metadata).to eq([{ location: 'one.html' }, { location: 'two.html' }])
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
it 'replays events when a subscriber joins during a result callback' do
|
|
161
|
+
stub_request(:any, url).to_return(status: 200)
|
|
162
|
+
checker.on(:result) { |result| checker.check(url, location: 'two.html') if result.options[:location] == 'one.html' }
|
|
163
|
+
checker.check(url, location: 'one.html')
|
|
164
|
+
checker.run
|
|
165
|
+
expect(checker.results[:success].map(&:options)).to eq([{ location: 'one.html' }, { location: 'two.html' }])
|
|
166
|
+
expect(a_request(:any, url)).to have_been_made.once
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it 'does not memoize a response if settings changed while queued' do
|
|
170
|
+
stub_request(:any, url).to_return(status: 200)
|
|
171
|
+
checker.check(url)
|
|
172
|
+
checker.headers = { 'Accept' => 'text/html' }
|
|
173
|
+
checker.run
|
|
174
|
+
checker.headers = {}
|
|
175
|
+
checker.check(url)
|
|
176
|
+
checker.run
|
|
177
|
+
expect(a_request(:any, url)).to have_been_made.twice
|
|
178
|
+
end
|
|
179
|
+
end
|
|
67
180
|
end
|
data/spec/spec_helper.rb
CHANGED
|
@@ -1,4 +1,244 @@
|
|
|
1
1
|
shared_context 'a link checker' do
|
|
2
|
+
context 'with redirects', :aggregate_failures do
|
|
3
|
+
let(:checker) { described_class.new(methods: ['GET'], max_redirects: 2) }
|
|
4
|
+
let(:url) { 'https://www.example.org/start' }
|
|
5
|
+
let(:result) { checker.results.values.flatten.first }
|
|
6
|
+
|
|
7
|
+
around do |example|
|
|
8
|
+
VCR.turned_off { example.run }
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
it 'resolves each relative redirect against the current URL' do
|
|
12
|
+
stub_request(:any, url).to_return(status: 302, headers: { 'Location' => '/nested/next' })
|
|
13
|
+
stub_request(:any, 'https://www.example.org/nested/next').to_return(status: 302, headers: { 'Location' => 'final' })
|
|
14
|
+
stub_request(:any, 'https://www.example.org/nested/final').to_return(status: 200)
|
|
15
|
+
checker.check(url, location: 'page.html')
|
|
16
|
+
checker.run if checker.respond_to?(:run)
|
|
17
|
+
expect(result.uri.to_s).to eq(url)
|
|
18
|
+
expect(result.result_uri.to_s).to eq('https://www.example.org/nested/final')
|
|
19
|
+
expect(result.options).to eq(location: 'page.html')
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
it 'stops redirect chains at the configured limit' do
|
|
23
|
+
stub_request(:any, url).to_return(status: 302, headers: { 'Location' => '/one' })
|
|
24
|
+
stub_request(:any, 'https://www.example.org/one').to_return(status: 302, headers: { 'Location' => '/two' })
|
|
25
|
+
stub_request(:any, 'https://www.example.org/two').to_return(status: 302, headers: { 'Location' => '/three' })
|
|
26
|
+
checker.check(url)
|
|
27
|
+
checker.run if checker.respond_to?(:run)
|
|
28
|
+
expect(result.error).to be_a(LinkChecker::Errors::RedirectLimitError)
|
|
29
|
+
expect(result.uri.to_s).to eq(url)
|
|
30
|
+
expect(result.result_uri.to_s).to eq('https://www.example.org/two')
|
|
31
|
+
expect(a_request(:any, 'https://www.example.org/three')).not_to have_been_made
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
it 'detects a redirect back to the original URL without requesting it again' do
|
|
35
|
+
stub_request(:any, url).to_return(status: 302, headers: { 'Location' => '/one' })
|
|
36
|
+
stub_request(:any, 'https://www.example.org/one').to_return(status: 302, headers: { 'Location' => url })
|
|
37
|
+
checker.check(url)
|
|
38
|
+
checker.run if checker.respond_to?(:run)
|
|
39
|
+
expect(checker.results[:error].first.error).to be_a(LinkChecker::Errors::RedirectLoopError)
|
|
40
|
+
expect(a_request(:any, url)).to have_been_made.once
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
it 'allows redirects to be disabled' do
|
|
44
|
+
checker.max_redirects = 0
|
|
45
|
+
stub_request(:any, url).to_return(status: 302, headers: { 'Location' => '/one' })
|
|
46
|
+
checker.check(url)
|
|
47
|
+
checker.run if checker.respond_to?(:run)
|
|
48
|
+
expect(checker.results[:error].first.error).to be_a(LinkChecker::Errors::RedirectLimitError)
|
|
49
|
+
expect(a_request(:any, 'https://www.example.org/one')).not_to have_been_made
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
context 'with transient retries' do
|
|
54
|
+
let(:checker_options) { {} }
|
|
55
|
+
let(:checker) { described_class.new({ methods: ['GET'], retries: 2 }.merge(checker_options)) }
|
|
56
|
+
let(:url) { 'https://www.example.org/retry' }
|
|
57
|
+
|
|
58
|
+
around do |example|
|
|
59
|
+
VCR.turned_off { example.run }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
it 'retries service errors and preserves metadata' do
|
|
63
|
+
stub_request(:get, url).to_return({ status: 503 }, { status: 200 })
|
|
64
|
+
checker.check(url, location: 'page.html')
|
|
65
|
+
checker.run if checker.respond_to?(:run)
|
|
66
|
+
expect(checker.results[:success].first.options).to eq(location: 'page.html')
|
|
67
|
+
expect(a_request(:get, url)).to have_been_made.twice
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
it 'does not retry permanent HTTP failures' do
|
|
71
|
+
stub_request(:get, url).to_return(status: 404)
|
|
72
|
+
checker.check(url)
|
|
73
|
+
checker.run if checker.respond_to?(:run)
|
|
74
|
+
expect(checker.results[:failure].first.code).to eq(404)
|
|
75
|
+
expect(a_request(:get, url)).to have_been_made.once
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
it 'retries timeouts and reports a timeout category' do
|
|
79
|
+
stub_request(:get, url).to_timeout
|
|
80
|
+
checker.check(url)
|
|
81
|
+
checker.run if checker.respond_to?(:run)
|
|
82
|
+
expect(checker.results[:error].first.error_type).to eq(:timeout)
|
|
83
|
+
expect(a_request(:get, url)).to have_been_made.times(3)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
it 'does not retry malformed URLs' do
|
|
87
|
+
checker.check('\/invalid-url')
|
|
88
|
+
checker.run if checker.respond_to?(:run)
|
|
89
|
+
expect(checker.results[:error].first.error_type).to eq(:other)
|
|
90
|
+
expect(checker.results[:error].size).to eq(1)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
context 'with a configured delay and backoff' do
|
|
94
|
+
let(:checker_options) { { retry_after: 0.005, retry_backoff: 2 } }
|
|
95
|
+
|
|
96
|
+
it 'waits before each retry without changing the retry budget' do
|
|
97
|
+
stub_request(:get, url).to_return({ status: 503 }, { status: 503 }, { status: 200 })
|
|
98
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
99
|
+
checker.check(url)
|
|
100
|
+
checker.run if checker.respond_to?(:run)
|
|
101
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
102
|
+
expect(elapsed).to be >= 0.015
|
|
103
|
+
expect(a_request(:get, url)).to have_been_made.times(3)
|
|
104
|
+
expect(checker.results[:success].size).to eq(1)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
context 'with Retry-After enabled' do
|
|
109
|
+
let(:checker_options) { { respect_retry_after: true, max_retry_delay: 0.005 } }
|
|
110
|
+
|
|
111
|
+
it 'honors a bounded server delay' do
|
|
112
|
+
stub_request(:get, url).to_return({ status: 429, headers: { 'Retry-After' => '5' } }, { status: 200 })
|
|
113
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
114
|
+
checker.check(url)
|
|
115
|
+
checker.run if checker.respond_to?(:run)
|
|
116
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
117
|
+
expect(elapsed).to be >= 0.005
|
|
118
|
+
expect(elapsed).to be < 1
|
|
119
|
+
expect(checker.results[:success].size).to eq(1)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
it 'accepts HTTP dates' do
|
|
123
|
+
stub_request(:get, url).to_return(
|
|
124
|
+
{ status: 503, headers: { 'Retry-After' => (Time.now + 60).httpdate } }, { status: 200 }
|
|
125
|
+
)
|
|
126
|
+
checker.check(url)
|
|
127
|
+
checker.run if checker.respond_to?(:run)
|
|
128
|
+
expect(checker.results[:success].size).to eq(1)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
it 'ignores invalid server delays' do
|
|
132
|
+
stub_request(:get, url).to_return({ status: 503, headers: { 'Retry-After' => 'invalid' } }, { status: 200 })
|
|
133
|
+
checker.check(url)
|
|
134
|
+
checker.run if checker.respond_to?(:run)
|
|
135
|
+
expect(checker.results[:success].size).to eq(1)
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
context 'with deduplication' do
|
|
141
|
+
let(:checker) { described_class.new(methods: ['GET'], deduplicate: true) }
|
|
142
|
+
let(:url) { 'https://www.example.org/' }
|
|
143
|
+
|
|
144
|
+
around do |example|
|
|
145
|
+
VCR.turned_off { example.run }
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it 'checks once and preserves metadata for every submission' do
|
|
149
|
+
stub_request(:any, url).to_return(status: 200)
|
|
150
|
+
checker.check(url, location: 'one.html')
|
|
151
|
+
checker.check(url, location: 'two.html')
|
|
152
|
+
checker.run if checker.respond_to?(:run)
|
|
153
|
+
expect(a_request(:any, url)).to have_been_made.once
|
|
154
|
+
expect(checker.results[:success].map(&:options)).to eq([{ location: 'one.html' }, { location: 'two.html' }])
|
|
155
|
+
expect(checker.results[:success].map(&:uri)).to eq([URI(url), URI(url)])
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
it 'rechecks failures instead of retaining them' do
|
|
159
|
+
stub_request(:any, url).to_return({ status: 404 }, { status: 200 })
|
|
160
|
+
checker.check(url)
|
|
161
|
+
checker.run if checker.respond_to?(:run)
|
|
162
|
+
checker.check(url)
|
|
163
|
+
checker.run if checker.respond_to?(:run)
|
|
164
|
+
expect(a_request(:any, url)).to have_been_made.twice
|
|
165
|
+
expect(checker.results[:failure].size).to eq(1)
|
|
166
|
+
expect(checker.results[:success].size).to eq(1)
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it 'keeps query values distinct' do
|
|
170
|
+
stub_request(:any, "#{url}?id=1").to_return(status: 200)
|
|
171
|
+
stub_request(:any, "#{url}?id=2").to_return(status: 404)
|
|
172
|
+
checker.check("#{url}?id=1")
|
|
173
|
+
checker.check("#{url}?id=2")
|
|
174
|
+
checker.run if checker.respond_to?(:run)
|
|
175
|
+
expect(checker.results[:success].first.uri.query).to eq('id=1')
|
|
176
|
+
expect(checker.results[:failure].first.uri.query).to eq('id=2')
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
it 'includes headers in request identity' do
|
|
180
|
+
stub_request(:any, url).to_return(status: 200)
|
|
181
|
+
checker.check(url)
|
|
182
|
+
checker.run if checker.respond_to?(:run)
|
|
183
|
+
checker.headers = { 'Accept' => 'text/html' }
|
|
184
|
+
checker.check(url)
|
|
185
|
+
checker.run if checker.respond_to?(:run)
|
|
186
|
+
expect(a_request(:any, url)).to have_been_made.twice
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
context 'without result collection' do
|
|
190
|
+
let(:checker) { described_class.new(methods: ['GET'], deduplicate: true, results: false) }
|
|
191
|
+
|
|
192
|
+
it 'still delivers results to every subscriber' do
|
|
193
|
+
completed = []
|
|
194
|
+
stub_request(:any, url).to_return(status: 200)
|
|
195
|
+
checker.on(:success) { |result| completed << result.options }
|
|
196
|
+
checker.check(url, location: 'one.html')
|
|
197
|
+
checker.check(url, location: 'two.html')
|
|
198
|
+
checker.run if checker.respond_to?(:run)
|
|
199
|
+
expect(completed).to eq([{ location: 'one.html' }, { location: 'two.html' }])
|
|
200
|
+
expect(checker.results).to be_nil
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
context 'with persistent caching' do
|
|
206
|
+
let(:directory) { Dir.mktmpdir('link-checker-spec-') }
|
|
207
|
+
let(:cache_options) { { file: File.join(directory, 'cache.json'), ttl: 60 } }
|
|
208
|
+
let(:checker) { described_class.new(methods: ['GET'], cache: cache_options) }
|
|
209
|
+
let(:next_checker) { described_class.new(methods: ['GET'], cache: cache_options) }
|
|
210
|
+
let(:url) { 'https://www.example.org/' }
|
|
211
|
+
|
|
212
|
+
around do |example|
|
|
213
|
+
VCR.turned_off { example.run }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
after do
|
|
217
|
+
FileUtils.remove_entry(directory)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
it 'reuses a success across checker instances with fresh metadata' do
|
|
221
|
+
stub_request(:any, url).to_return(status: 200)
|
|
222
|
+
checker.check(url, location: 'one.html')
|
|
223
|
+
checker.run if checker.respond_to?(:run)
|
|
224
|
+
next_checker.check(url, location: 'two.html')
|
|
225
|
+
next_checker.run if next_checker.respond_to?(:run)
|
|
226
|
+
expect(next_checker.results[:success].first.cached?).to be true
|
|
227
|
+
expect(next_checker.results[:success].first.options).to eq(location: 'two.html')
|
|
228
|
+
expect(a_request(:any, url)).to have_been_made.once
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
it 'does not persist failed checks' do
|
|
232
|
+
stub_request(:any, url).to_return({ status: 404 }, { status: 200 })
|
|
233
|
+
checker.check(url)
|
|
234
|
+
checker.run if checker.respond_to?(:run)
|
|
235
|
+
next_checker.check(url)
|
|
236
|
+
next_checker.run if next_checker.respond_to?(:run)
|
|
237
|
+
expect(next_checker.results[:success].first.cached?).to be false
|
|
238
|
+
expect(a_request(:any, url)).to have_been_made.twice
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
2
242
|
context 'default' do
|
|
3
243
|
subject do
|
|
4
244
|
described_class.new
|
|
@@ -7,6 +247,10 @@ shared_context 'a link checker' do
|
|
|
7
247
|
it 'has a default user-agent' do
|
|
8
248
|
expect(subject.user_agent).to eq "Ruby Link Checker/#{LinkChecker::VERSION}"
|
|
9
249
|
end
|
|
250
|
+
|
|
251
|
+
it 'has no default headers' do
|
|
252
|
+
expect(subject.headers).to eq({})
|
|
253
|
+
end
|
|
10
254
|
end
|
|
11
255
|
|
|
12
256
|
context 'user-agent' do
|
|
@@ -19,6 +263,16 @@ shared_context 'a link checker' do
|
|
|
19
263
|
end
|
|
20
264
|
end
|
|
21
265
|
|
|
266
|
+
context 'headers' do
|
|
267
|
+
subject do
|
|
268
|
+
described_class.new(headers: { 'DNT' => '1' })
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
it 'updates headers' do
|
|
272
|
+
expect(subject.headers).to eq('DNT' => '1')
|
|
273
|
+
end
|
|
274
|
+
end
|
|
275
|
+
|
|
22
276
|
context 'check' do
|
|
23
277
|
let(:url) { 'https://www.example.org' }
|
|
24
278
|
|
|
@@ -104,6 +358,16 @@ shared_context 'a link checker' do
|
|
|
104
358
|
expect(result.options).to eq({})
|
|
105
359
|
end
|
|
106
360
|
|
|
361
|
+
context 'with custom headers' do
|
|
362
|
+
subject do
|
|
363
|
+
described_class.new(methods: ['GET'], headers: { 'DNT' => '1' })
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
it 'sends custom headers' do
|
|
367
|
+
expect(result.request_headers['DNT']).to eq '1'
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
|
|
107
371
|
it 'returns results' do
|
|
108
372
|
expect(subject.results).to eq(
|
|
109
373
|
error: [],
|
|
@@ -152,7 +416,7 @@ shared_context 'a link checker' do
|
|
|
152
416
|
|
|
153
417
|
context 'with 1 retry' do
|
|
154
418
|
subject do
|
|
155
|
-
described_class.new(methods: ['GET'], retries: 1)
|
|
419
|
+
described_class.new(methods: ['GET'], retries: 1, retry_status_codes: nil)
|
|
156
420
|
end
|
|
157
421
|
|
|
158
422
|
it 'fails' do
|
|
@@ -168,7 +432,7 @@ shared_context 'a link checker' do
|
|
|
168
432
|
|
|
169
433
|
context 'with 2 retries' do
|
|
170
434
|
subject do
|
|
171
|
-
described_class.new(methods: ['GET'], retries: 2)
|
|
435
|
+
described_class.new(methods: ['GET'], retries: 2, retry_status_codes: nil)
|
|
172
436
|
end
|
|
173
437
|
|
|
174
438
|
it 'fails' do
|
|
@@ -216,7 +480,7 @@ shared_context 'a link checker' do
|
|
|
216
480
|
expect(result.error?).to be true
|
|
217
481
|
expect(result.error).to be_a LinkChecker::Errors::RedirectLoopError
|
|
218
482
|
expect(result.redirect?).to be false
|
|
219
|
-
expect(subject).to have_received(:called!).with(:redirect, anything).
|
|
483
|
+
expect(subject).to have_received(:called!).with(:redirect, anything).once
|
|
220
484
|
expect(subject).to have_received(:called!).with(:error, result)
|
|
221
485
|
expect(subject).not_to have_received(:called!).with(:failure, result)
|
|
222
486
|
expect(subject).not_to have_received(:called!).with(:success, result)
|
metadata
CHANGED
|
@@ -1,15 +1,29 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ruby-link-checker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Daniel Doubrovkine
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-11-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: async-http
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0'
|
|
20
|
+
type: :development
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0'
|
|
13
27
|
- !ruby/object:Gem::Dependency
|
|
14
28
|
name: typhoeus
|
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -30,11 +44,21 @@ executables: []
|
|
|
30
44
|
extensions: []
|
|
31
45
|
extra_rdoc_files: []
|
|
32
46
|
files:
|
|
47
|
+
- ".github/workflows/coverage.yml"
|
|
48
|
+
- ".github/workflows/danger-comment.yml"
|
|
49
|
+
- ".github/workflows/danger.yml"
|
|
50
|
+
- ".github/workflows/lint.yml"
|
|
51
|
+
- ".github/workflows/release.yml"
|
|
52
|
+
- ".github/workflows/test.yml"
|
|
53
|
+
- ".gitignore"
|
|
54
|
+
- ".rspec"
|
|
55
|
+
- ".rubocop.yml"
|
|
56
|
+
- ".rubocop_todo.yml"
|
|
57
|
+
- AGENTS.md
|
|
33
58
|
- CHANGELOG.md
|
|
34
59
|
- CONTRIBUTING.md
|
|
35
60
|
- Dangerfile
|
|
36
61
|
- Gemfile
|
|
37
|
-
- Gemfile.lock
|
|
38
62
|
- LICENSE.md
|
|
39
63
|
- README.md
|
|
40
64
|
- RELEASING.md
|
|
@@ -42,42 +66,19 @@ files:
|
|
|
42
66
|
- benchmarks/check.rb
|
|
43
67
|
- benchmarks/data/opensearch.org/links.txt
|
|
44
68
|
- benchmarks/data/opensearch.org/small.txt
|
|
45
|
-
- coverage/assets/0.13.1/DataTables-1.10.20/images/sort_asc.png
|
|
46
|
-
- coverage/assets/0.13.1/DataTables-1.10.20/images/sort_asc_disabled.png
|
|
47
|
-
- coverage/assets/0.13.1/DataTables-1.10.20/images/sort_both.png
|
|
48
|
-
- coverage/assets/0.13.1/DataTables-1.10.20/images/sort_desc.png
|
|
49
|
-
- coverage/assets/0.13.1/DataTables-1.10.20/images/sort_desc_disabled.png
|
|
50
|
-
- coverage/assets/0.13.1/application.css
|
|
51
|
-
- coverage/assets/0.13.1/application.js
|
|
52
|
-
- coverage/assets/0.13.1/colorbox/border.png
|
|
53
|
-
- coverage/assets/0.13.1/colorbox/controls.png
|
|
54
|
-
- coverage/assets/0.13.1/colorbox/loading.gif
|
|
55
|
-
- coverage/assets/0.13.1/colorbox/loading_background.png
|
|
56
|
-
- coverage/assets/0.13.1/favicon_green.png
|
|
57
|
-
- coverage/assets/0.13.1/favicon_red.png
|
|
58
|
-
- coverage/assets/0.13.1/favicon_yellow.png
|
|
59
|
-
- coverage/assets/0.13.1/images/ui-bg_flat_0_aaaaaa_40x100.png
|
|
60
|
-
- coverage/assets/0.13.1/images/ui-bg_flat_75_ffffff_40x100.png
|
|
61
|
-
- coverage/assets/0.13.1/images/ui-bg_glass_55_fbf9ee_1x400.png
|
|
62
|
-
- coverage/assets/0.13.1/images/ui-bg_glass_65_ffffff_1x400.png
|
|
63
|
-
- coverage/assets/0.13.1/images/ui-bg_glass_75_dadada_1x400.png
|
|
64
|
-
- coverage/assets/0.13.1/images/ui-bg_glass_75_e6e6e6_1x400.png
|
|
65
|
-
- coverage/assets/0.13.1/images/ui-bg_glass_95_fef1ec_1x400.png
|
|
66
|
-
- coverage/assets/0.13.1/images/ui-bg_highlight-soft_75_cccccc_1x100.png
|
|
67
|
-
- coverage/assets/0.13.1/images/ui-icons_222222_256x240.png
|
|
68
|
-
- coverage/assets/0.13.1/images/ui-icons_2e83ff_256x240.png
|
|
69
|
-
- coverage/assets/0.13.1/images/ui-icons_454545_256x240.png
|
|
70
|
-
- coverage/assets/0.13.1/images/ui-icons_888888_256x240.png
|
|
71
|
-
- coverage/assets/0.13.1/images/ui-icons_cd0a0a_256x240.png
|
|
72
|
-
- coverage/assets/0.13.1/loading.gif
|
|
73
|
-
- coverage/assets/0.13.1/magnify.png
|
|
74
|
-
- coverage/index.html
|
|
75
69
|
- lib/ruby-link-checker.rb
|
|
70
|
+
- lib/ruby-link-checker/async/http.rb
|
|
71
|
+
- lib/ruby-link-checker/async/http/checker.rb
|
|
72
|
+
- lib/ruby-link-checker/async/http/config.rb
|
|
73
|
+
- lib/ruby-link-checker/async/http/result.rb
|
|
74
|
+
- lib/ruby-link-checker/cache.rb
|
|
76
75
|
- lib/ruby-link-checker/callbacks.rb
|
|
77
76
|
- lib/ruby-link-checker/checker.rb
|
|
78
77
|
- lib/ruby-link-checker/config.rb
|
|
79
78
|
- lib/ruby-link-checker/errors.rb
|
|
80
79
|
- lib/ruby-link-checker/errors/base_error.rb
|
|
80
|
+
- lib/ruby-link-checker/errors/connection_error.rb
|
|
81
|
+
- lib/ruby-link-checker/errors/redirect_limit_error.rb
|
|
81
82
|
- lib/ruby-link-checker/errors/redirect_loop_error.rb
|
|
82
83
|
- lib/ruby-link-checker/logger.rb
|
|
83
84
|
- lib/ruby-link-checker/net/http.rb
|
|
@@ -105,10 +106,16 @@ files:
|
|
|
105
106
|
- spec/fixtures/404_404.yml
|
|
106
107
|
- spec/fixtures/429_200.yml
|
|
107
108
|
- spec/fixtures/429_429_200.yml
|
|
109
|
+
- spec/ruby-link-checker/async/http/checker_spec.rb
|
|
108
110
|
- spec/ruby-link-checker/callbacks_spec.rb
|
|
109
111
|
- spec/ruby-link-checker/checker_spec.rb
|
|
110
112
|
- spec/ruby-link-checker/config_spec.rb
|
|
113
|
+
- spec/ruby-link-checker/errors/redirect_loop_error_spec.rb
|
|
111
114
|
- spec/ruby-link-checker/net/http/checker_spec.rb
|
|
115
|
+
- spec/ruby-link-checker/result_error_spec.rb
|
|
116
|
+
- spec/ruby-link-checker/result_spec.rb
|
|
117
|
+
- spec/ruby-link-checker/task_spec.rb
|
|
118
|
+
- spec/ruby-link-checker/tasks_spec.rb
|
|
112
119
|
- spec/ruby-link-checker/typhoeus/hydra/checker_spec.rb
|
|
113
120
|
- spec/ruby-link-checker/version_spec.rb
|
|
114
121
|
- spec/spec_helper.rb
|
|
@@ -130,14 +137,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
130
137
|
requirements:
|
|
131
138
|
- - ">="
|
|
132
139
|
- !ruby/object:Gem::Version
|
|
133
|
-
version: '
|
|
140
|
+
version: '3.3'
|
|
134
141
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
135
142
|
requirements:
|
|
136
143
|
- - ">="
|
|
137
144
|
- !ruby/object:Gem::Version
|
|
138
145
|
version: 1.3.6
|
|
139
146
|
requirements: []
|
|
140
|
-
rubygems_version: 3.5.
|
|
147
|
+
rubygems_version: 3.5.22
|
|
141
148
|
signing_key:
|
|
142
149
|
specification_version: 4
|
|
143
150
|
summary: Fast ruby link checker.
|