ruby-link-checker 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/coverage.yml +24 -0
- data/.github/workflows/danger-comment.yml +17 -0
- data/.github/workflows/danger.yml +12 -0
- data/.github/workflows/lint.yml +20 -0
- data/.github/workflows/release.yml +14 -0
- data/.github/workflows/test.yml +25 -0
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/.rubocop.yml +26 -0
- data/.rubocop_todo.yml +332 -0
- data/AGENTS.md +23 -0
- data/CHANGELOG.md +15 -0
- data/CONTRIBUTING.md +4 -0
- data/Dangerfile +0 -0
- data/Gemfile +3 -0
- data/LICENSE.md +0 -0
- data/README.md +78 -4
- data/RELEASING.md +17 -40
- data/Rakefile +8 -0
- data/benchmarks/check.rb +0 -0
- data/benchmarks/data/opensearch.org/links.txt +0 -0
- data/benchmarks/data/opensearch.org/small.txt +0 -0
- data/lib/ruby-link-checker/async/http/checker.rb +32 -6
- data/lib/ruby-link-checker/async/http/config.rb +4 -0
- data/lib/ruby-link-checker/async/http/result.rb +4 -0
- data/lib/ruby-link-checker/async/http.rb +0 -0
- data/lib/ruby-link-checker/cache.rb +101 -0
- data/lib/ruby-link-checker/callbacks.rb +0 -0
- data/lib/ruby-link-checker/checker.rb +76 -4
- data/lib/ruby-link-checker/config.rb +62 -0
- data/lib/ruby-link-checker/errors/base_error.rb +0 -0
- data/lib/ruby-link-checker/errors/connection_error.rb +23 -0
- data/lib/ruby-link-checker/errors/redirect_limit_error.rb +15 -0
- data/lib/ruby-link-checker/errors/redirect_loop_error.rb +0 -0
- data/lib/ruby-link-checker/errors.rb +2 -0
- data/lib/ruby-link-checker/logger.rb +0 -0
- data/lib/ruby-link-checker/net/http/checker.rb +2 -1
- data/lib/ruby-link-checker/net/http/config.rb +0 -0
- data/lib/ruby-link-checker/net/http/result.rb +4 -0
- data/lib/ruby-link-checker/net/http.rb +0 -0
- data/lib/ruby-link-checker/result.rb +25 -0
- data/lib/ruby-link-checker/task.rb +2 -0
- data/lib/ruby-link-checker/tasks.rb +40 -6
- data/lib/ruby-link-checker/typhoeus/hydra/checker.rb +27 -5
- data/lib/ruby-link-checker/typhoeus/hydra/config.rb +0 -0
- data/lib/ruby-link-checker/typhoeus/hydra/result.rb +4 -0
- data/lib/ruby-link-checker/typhoeus/hydra.rb +0 -0
- data/lib/ruby-link-checker/version.rb +1 -1
- data/lib/ruby-link-checker.rb +1 -0
- data/lib/ruby_link_checker.rb +0 -0
- data/ruby-link-checker.gemspec +1 -1
- data/spec/fixtures/200.yml +0 -0
- data/spec/fixtures/301_200.yml +0 -0
- data/spec/fixtures/301_301.yml +0 -0
- data/spec/fixtures/301_400_301_200.yml +0 -0
- data/spec/fixtures/301_403.yml +0 -0
- data/spec/fixtures/308_429.yml +0 -0
- data/spec/fixtures/404.yml +0 -0
- data/spec/fixtures/404_200.yml +0 -0
- data/spec/fixtures/404_404.yml +0 -0
- data/spec/fixtures/429_200.yml +0 -0
- data/spec/fixtures/429_429_200.yml +0 -0
- data/spec/ruby-link-checker/async/http/checker_spec.rb +86 -1
- data/spec/ruby-link-checker/callbacks_spec.rb +0 -0
- data/spec/ruby-link-checker/checker_spec.rb +89 -2
- data/spec/ruby-link-checker/config_spec.rb +6 -0
- data/spec/ruby-link-checker/errors/redirect_loop_error_spec.rb +0 -0
- data/spec/ruby-link-checker/net/http/checker_spec.rb +15 -0
- data/spec/ruby-link-checker/result_error_spec.rb +17 -0
- data/spec/ruby-link-checker/result_spec.rb +0 -0
- data/spec/ruby-link-checker/task_spec.rb +0 -0
- data/spec/ruby-link-checker/tasks_spec.rb +44 -0
- data/spec/ruby-link-checker/typhoeus/hydra/checker_spec.rb +88 -1
- data/spec/ruby-link-checker/version_spec.rb +0 -0
- data/spec/spec_helper.rb +0 -0
- data/spec/support/config.rb +0 -0
- data/spec/support/link_checker.rb +243 -3
- data/spec/support/vcr.rb +0 -0
- data/spec/support/with_result.rb +0 -0
- data/spec/support/with_url.rb +0 -0
- metadata +21 -34
- data/Gemfile.lock +0 -334
- data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_asc.png +0 -0
- data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_asc_disabled.png +0 -0
- data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_both.png +0 -0
- data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_desc.png +0 -0
- data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_desc_disabled.png +0 -0
- data/coverage/assets/0.13.2/application.css +0 -1
- data/coverage/assets/0.13.2/application.js +0 -7
- data/coverage/assets/0.13.2/colorbox/border.png +0 -0
- data/coverage/assets/0.13.2/colorbox/controls.png +0 -0
- data/coverage/assets/0.13.2/colorbox/loading.gif +0 -0
- data/coverage/assets/0.13.2/colorbox/loading_background.png +0 -0
- data/coverage/assets/0.13.2/favicon_green.png +0 -0
- data/coverage/assets/0.13.2/favicon_red.png +0 -0
- data/coverage/assets/0.13.2/favicon_yellow.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_flat_0_aaaaaa_40x100.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_flat_75_ffffff_40x100.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_glass_55_fbf9ee_1x400.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_glass_65_ffffff_1x400.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_glass_75_dadada_1x400.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_glass_95_fef1ec_1x400.png +0 -0
- data/coverage/assets/0.13.2/images/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
- data/coverage/assets/0.13.2/images/ui-icons_222222_256x240.png +0 -0
- data/coverage/assets/0.13.2/images/ui-icons_2e83ff_256x240.png +0 -0
- data/coverage/assets/0.13.2/images/ui-icons_454545_256x240.png +0 -0
- data/coverage/assets/0.13.2/images/ui-icons_888888_256x240.png +0 -0
- data/coverage/assets/0.13.2/images/ui-icons_cd0a0a_256x240.png +0 -0
- data/coverage/assets/0.13.2/loading.gif +0 -0
- data/coverage/assets/0.13.2/magnify.png +0 -0
- data/coverage/index.html +0 -10746
data/README.md
CHANGED
|
@@ -19,9 +19,13 @@ A fast Ruby link checker with support for multiple HTTP libraries. Does not pars
|
|
|
19
19
|
- [LinkChecker::Net::HTTP](#linkcheckernethttp)
|
|
20
20
|
- [Options](#options)
|
|
21
21
|
- [Retries](#retries)
|
|
22
|
+
- [Redirects](#redirects)
|
|
22
23
|
- [Retry After](#retry-after)
|
|
24
|
+
- [Retry Policy](#retry-policy)
|
|
23
25
|
- [Results](#results)
|
|
24
26
|
- [Methods](#methods)
|
|
27
|
+
- [Deduplication](#deduplication)
|
|
28
|
+
- [Caching](#caching)
|
|
25
29
|
- [Logger](#logger)
|
|
26
30
|
- [User-Agent](#user-agent)
|
|
27
31
|
- [Headers](#headers)
|
|
@@ -172,6 +176,20 @@ LinkChecker::Async::HTTP.configure do |config|
|
|
|
172
176
|
end
|
|
173
177
|
```
|
|
174
178
|
|
|
179
|
+
You can also force a specific HTTP protocol version, `:http1` or `:http2`, per instance or globally. By default, `async-http` negotiates the protocol version with the server.
|
|
180
|
+
|
|
181
|
+
```ruby
|
|
182
|
+
LinkChecker::Async::HTTP.configure do |config|
|
|
183
|
+
config.protocol = :http2
|
|
184
|
+
end
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Async::HTTP reuses connection-pooling clients per scheme/host/port. Its default concurrency is unlimited; set `max_concurrency` to bound active requests without counting delayed retries against that limit.
|
|
188
|
+
|
|
189
|
+
```ruby
|
|
190
|
+
checker = LinkChecker::Async::HTTP::Checker.new(max_concurrency: 25)
|
|
191
|
+
```
|
|
192
|
+
|
|
175
193
|
#### [LinkChecker::Net::HTTP](lib/ruby-link-checker/net/http/checker.rb)
|
|
176
194
|
|
|
177
195
|
Slow, sequential checker.
|
|
@@ -210,18 +228,43 @@ end
|
|
|
210
228
|
|
|
211
229
|
#### Retries
|
|
212
230
|
|
|
213
|
-
By default link checkers do not retry.
|
|
231
|
+
By default link checkers do not retry. Set `retries` to enable retries for transient connection errors/timeouts and HTTP statuses 408, 425, 429, 500, 502, 503, and 504. Permanent HTTP failures and invalid URLs are not retried. HEAD-to-GET fallback still applies independently of the retry policy. The retry budget is shared across methods: with `retries: 2`, repeated transient failures produce `HEAD`, `HEAD`, `HEAD`, then `GET`.
|
|
214
232
|
|
|
215
233
|
```ruby
|
|
216
|
-
checker = LinkChecker::Net::HTTP::Checker.new(
|
|
234
|
+
checker = LinkChecker::Net::HTTP::Checker.new(retries: 1)
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
#### Redirects
|
|
238
|
+
|
|
239
|
+
Checkers follow up to 20 redirects per attempt and detect redirect loops. Relative redirects are resolved against the current response URL. Set `max_redirects` to change the limit, or use `0` to disable redirects.
|
|
240
|
+
|
|
241
|
+
```ruby
|
|
242
|
+
checker = LinkChecker::Typhoeus::Hydra::Checker.new(max_redirects: 10)
|
|
217
243
|
```
|
|
218
244
|
|
|
219
245
|
#### Retry After
|
|
220
246
|
|
|
221
|
-
|
|
247
|
+
Set `retry_after` to a minimum delay in seconds (default `0`). Optional `retry_backoff` multiplies that delay on successive retries (default `1`), and `retry_jitter` adds a random fraction between `0` and the configured value (default `0`, maximum `1`). All delays are capped by `max_retry_delay` (default `60` seconds).
|
|
248
|
+
|
|
249
|
+
```ruby
|
|
250
|
+
checker = LinkChecker::Typhoeus::Hydra::Checker.new(
|
|
251
|
+
retries: 2,
|
|
252
|
+
retry_after: 1,
|
|
253
|
+
retry_backoff: 2,
|
|
254
|
+
retry_jitter: 0.1,
|
|
255
|
+
respect_retry_after: true,
|
|
256
|
+
max_retry_delay: 60
|
|
257
|
+
)
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
With `respect_retry_after: true` (default `false`), HTTP `Retry-After` delta-seconds and dates provide a minimum delay, subject to the same cap. Invalid or past dates are ignored. Async delays yield without blocking other requests. Hydra queues delayed retries after the active batch drains, so actual delays can exceed the configured minimum without blocking completion callbacks. Delays apply to retries, not ordinary HEAD-to-GET fallback.
|
|
261
|
+
|
|
262
|
+
#### Retry Policy
|
|
263
|
+
|
|
264
|
+
Customize `retry_status_codes` to choose retryable HTTP statuses. Set it to `nil` to restore the previous behavior of retrying every failure and error, including malformed URLs and programming errors.
|
|
222
265
|
|
|
223
266
|
```ruby
|
|
224
|
-
checker = LinkChecker::Net::HTTP::Checker.new(
|
|
267
|
+
checker = LinkChecker::Net::HTTP::Checker.new(retries: 1, retry_status_codes: [429, 503])
|
|
225
268
|
```
|
|
226
269
|
|
|
227
270
|
#### Results
|
|
@@ -256,6 +299,34 @@ The following examples disables `GET` and only makes `HEAD` requests.
|
|
|
256
299
|
checker = LinkChecker::Net::HTTP::Checker.new(methods: %w[HEAD])
|
|
257
300
|
```
|
|
258
301
|
|
|
302
|
+
#### Deduplication
|
|
303
|
+
|
|
304
|
+
Set `deduplicate: true` to coalesce identical in-flight requests and reuse successful results for the lifetime of the checker. Failures and errors are rechecked after completion. Each submission still receives its own result events and metadata, including events emitted before it joined an in-flight check. Deduplication is disabled by default.
|
|
305
|
+
|
|
306
|
+
```ruby
|
|
307
|
+
checker = LinkChecker::Typhoeus::Hydra::Checker.new(deduplicate: true)
|
|
308
|
+
checker.check('https://www.example.org', location: 'one.html')
|
|
309
|
+
checker.check('https://www.example.org', location: 'two.html')
|
|
310
|
+
checker.run # one check, two sets of result events
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
Identity includes the complete URL (including query values and fragment), transport, methods, headers, user-agent, and request-related settings. Changing those settings prevents reuse. URLs containing user-info credentials bypass deduplication and caching.
|
|
314
|
+
|
|
315
|
+
#### Caching
|
|
316
|
+
|
|
317
|
+
Persistent success caching is disabled by default. Enable it with an explicit file path and positive TTL in seconds. Recent successful checks can then be reused across checker instances; failures and errors are never stored. Cache writes are atomic and locked, and unreadable or malformed files fall back to normal checking.
|
|
318
|
+
|
|
319
|
+
```ruby
|
|
320
|
+
checker = LinkChecker::Typhoeus::Hydra::Checker.new(
|
|
321
|
+
deduplicate: true,
|
|
322
|
+
cache: { file: 'tmp/link-checker/cache.json', ttl: 3600 }
|
|
323
|
+
)
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
Each cache hit emits `:result` and `:success` with the new submission's metadata and `cached? == true`. Persistent hits return `LinkChecker::CachedResult` with the HTTP method, status, original/final URIs, and metadata; native request/response objects and headers are not persisted. Redirect targets containing user-info credentials are not written. Add the cache directory to your ignore rules.
|
|
327
|
+
|
|
328
|
+
Caching alone does not coalesce in-flight checks. With deduplication enabled, successful results remain reusable within that checker even after the persistent TTL expires. Cached successes do not replay historical redirect/retry events.
|
|
329
|
+
|
|
259
330
|
#### Logger
|
|
260
331
|
|
|
261
332
|
Pass your own logger.
|
|
@@ -334,6 +405,9 @@ Events are called with results, which contain the following properties.
|
|
|
334
405
|
| :request_headers | Request headers. |
|
|
335
406
|
| :redirect_to | A redirect URL in case of redirects. |
|
|
336
407
|
| :error | A raised error in case of errors. |
|
|
408
|
+
| :error_type | `:timeout`, `:connection`, or `:other` for errors; otherwise `nil`. |
|
|
409
|
+
| :response_headers | Response headers, accessible using `[]`. |
|
|
410
|
+
| :cached? | Whether this result was reused from a successful check. |
|
|
337
411
|
|
|
338
412
|
See [result.rb](lib/ruby-link-checker/result.rb) for more details.
|
|
339
413
|
|
data/RELEASING.md
CHANGED
|
@@ -2,59 +2,36 @@
|
|
|
2
2
|
|
|
3
3
|
There are no hard rules about when to release ruby-link-checker. Release bug fixes frequently, features not so frequently and breaking API changes rarely.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Releases use [ruby-gem-release](https://github.com/dblock/ruby-gem-release/tree/v0.3.1) and the manual [Release workflow](.github/workflows/release.yml). Merges do not publish. Every workflow dispatch performs a real release.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
## One-Time Setup
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
bundle install
|
|
11
|
-
rake
|
|
12
|
-
```
|
|
13
|
-
|
|
14
|
-
Check that the last build succeeded.
|
|
15
|
-
|
|
16
|
-
Add a date to this release in [CHANGELOG.md](CHANGELOG.md).
|
|
17
|
-
|
|
18
|
-
```
|
|
19
|
-
### 0.2.2 (2015/7/10)
|
|
20
|
-
```
|
|
9
|
+
Create a GitHub environment named `release` in this repository. Restrict deployment to `main` and require approval when available.
|
|
21
10
|
|
|
22
|
-
|
|
11
|
+
On [RubyGems.org](https://rubygems.org/gems/ruby-link-checker/trusted_publishers), configure a trusted publisher with repository owner `dblock`, repository name `ruby-link-checker`, workflow filename `release.yml`, and environment `release`. Set the separate workflow repository owner to `dblock` and name to `ruby-gem-release` because publication uses its reusable workflow. No stored RubyGems API key is needed.
|
|
23
12
|
|
|
24
|
-
|
|
13
|
+
The workflow requests `contents: write` and `id-token: write`. Branch rules must allow the release identity to push fast-forward release and next-version commits to `main` and create tags, without allowing force pushes or deletion. Adding the workflow does not configure these permissions or the trusted publisher.
|
|
25
14
|
|
|
26
|
-
|
|
27
|
-
git add README.md CHANGELOG.md lib/ruby-link-checker/version.rb
|
|
28
|
-
git commit -m "Preparing for release, 0.2.2."
|
|
29
|
-
```
|
|
15
|
+
## Prepare and Publish
|
|
30
16
|
|
|
31
|
-
|
|
17
|
+
Prepare the intended version in `lib/ruby-link-checker/version.rb` and its matching `### VERSION (Next)` section in `CHANGELOG.md` through a PR. Keep exactly one `* Your contribution here.` placeholder in that section. The workflow releases the current version, dates the changelog, and removes the placeholder; it does not select a new release version.
|
|
32
18
|
|
|
33
|
-
|
|
34
|
-
$ rake release
|
|
19
|
+
Run tests and RuboCop locally and review the release changes before merging:
|
|
35
20
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
Pushed git commits and tags.
|
|
39
|
-
Pushed ruby-link-checker 0.2.2 to rubygems.org.
|
|
21
|
+
```sh
|
|
22
|
+
bundle exec rake
|
|
40
23
|
```
|
|
41
24
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
Add the next release to [CHANGELOG.md](CHANGELOG.md).
|
|
25
|
+
After merging, explicit release approval, and successful Trusted Publishing setup, dispatch from `main`:
|
|
45
26
|
|
|
27
|
+
```sh
|
|
28
|
+
gh workflow run release.yml --ref main
|
|
46
29
|
```
|
|
47
|
-
### 0.2.3 (Next)
|
|
48
30
|
|
|
49
|
-
|
|
50
|
-
```
|
|
31
|
+
The workflow runs `bundle exec rake`, publishes the verified gem to RubyGems, pushes the release commit and tag atomically, and creates a GitHub Release with the gem attached. It then increments the patch version and pushes a new development commit with a fresh changelog section. Actions-token pushes do not trigger ordinary push workflows, so the release workflow runs tests and RuboCop before publication.
|
|
51
32
|
|
|
52
|
-
|
|
33
|
+
## Verify and Recover
|
|
53
34
|
|
|
54
|
-
|
|
35
|
+
Verify the published RubyGems version, matching tag and GitHub Release, attached gem, and next development version on `main`.
|
|
55
36
|
|
|
56
|
-
|
|
57
|
-
git add CHANGELOG.md lib/ruby-link-checker/version.rb
|
|
58
|
-
git commit -m "Preparing for next development iteration, 0.2.3."
|
|
59
|
-
git push origin main
|
|
60
|
-
```
|
|
37
|
+
Never automatically rerun a failed or timed-out publication: RubyGems may have accepted the upload. Check registry and GitHub state first. Do not republish an existing version or move its tag. Available `release.gem`, `release.json`, and `release.bundle` recovery artifacts are retained for 30 days; follow the [upstream recovery instructions](https://github.com/dblock/ruby-gem-release/blob/v0.3.1/RELEASING.md#verify-and-recover) to complete only the missing steps.
|
data/Rakefile
CHANGED
|
@@ -15,4 +15,12 @@ end
|
|
|
15
15
|
require 'rubocop/rake_task'
|
|
16
16
|
RuboCop::RakeTask.new(:rubocop)
|
|
17
17
|
|
|
18
|
+
require 'ruby-gem-release/workflow/ruby_gem'
|
|
19
|
+
require 'ruby-gem-release/changelog'
|
|
20
|
+
|
|
21
|
+
RubyGemRelease::Workflow::RubyGem.new(
|
|
22
|
+
version: RubyGemRelease::Version::RubyConstant.new(path: 'lib/ruby-link-checker/version.rb', constant: 'VERSION'),
|
|
23
|
+
changelog: RubyGemRelease::Changelog::Intridea.new(placeholder: '* Your contribution here.')
|
|
24
|
+
).install_tasks
|
|
25
|
+
|
|
18
26
|
task default: %i[rubocop spec]
|
data/benchmarks/check.rb
CHANGED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
@@ -3,7 +3,7 @@ module LinkChecker
|
|
|
3
3
|
module HTTP
|
|
4
4
|
class Task < ::LinkChecker::Task
|
|
5
5
|
def run!
|
|
6
|
-
checker._queue { _run! }
|
|
6
|
+
checker._queue(delay: delay) { _run! }
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
private
|
|
@@ -42,25 +42,36 @@ module LinkChecker
|
|
|
42
42
|
end
|
|
43
43
|
@queue = []
|
|
44
44
|
@clients = {}
|
|
45
|
+
if max_concurrency && (!max_concurrency.is_a?(Integer) || max_concurrency < 1)
|
|
46
|
+
raise ArgumentError, "Invalid concurrency limit: #{max_concurrency}"
|
|
47
|
+
end
|
|
48
|
+
|
|
45
49
|
super
|
|
46
50
|
end
|
|
47
51
|
|
|
48
52
|
# Runs all queued checks concurrently and blocks until they complete.
|
|
49
53
|
def run
|
|
54
|
+
require 'async/semaphore'
|
|
55
|
+
|
|
50
56
|
Barrier do |barrier|
|
|
51
57
|
@barrier = barrier
|
|
58
|
+
@semaphore = ::Async::Semaphore.new(max_concurrency) if max_concurrency
|
|
52
59
|
_flush!
|
|
53
60
|
end
|
|
54
61
|
ensure
|
|
55
62
|
@barrier = nil
|
|
63
|
+
@semaphore = nil
|
|
56
64
|
_close_clients!
|
|
57
65
|
end
|
|
58
66
|
|
|
59
|
-
def _queue(&block)
|
|
67
|
+
def _queue(delay: 0, &block)
|
|
60
68
|
if @barrier
|
|
61
|
-
@barrier.async
|
|
69
|
+
@barrier.async do
|
|
70
|
+
sleep(delay) if delay.positive?
|
|
71
|
+
@semaphore ? @semaphore.acquire(&block) : block.call
|
|
72
|
+
end
|
|
62
73
|
else
|
|
63
|
-
@queue << block
|
|
74
|
+
@queue << [delay, block]
|
|
64
75
|
end
|
|
65
76
|
end
|
|
66
77
|
|
|
@@ -69,17 +80,32 @@ module LinkChecker
|
|
|
69
80
|
def _client_for(uri)
|
|
70
81
|
key = [uri.scheme, uri.host, uri.port]
|
|
71
82
|
@clients[key] ||= begin
|
|
72
|
-
endpoint = ::Async::HTTP::Endpoint.parse(uri.to_s, timeout: open_timeout)
|
|
83
|
+
endpoint = ::Async::HTTP::Endpoint.parse(uri.to_s, timeout: open_timeout, **_endpoint_options)
|
|
73
84
|
::Async::HTTP::Client.new(endpoint)
|
|
74
85
|
end
|
|
75
86
|
end
|
|
76
87
|
|
|
77
88
|
private
|
|
78
89
|
|
|
90
|
+
# Maps the configured protocol (:http1 or :http2) to the
|
|
91
|
+
# corresponding async-http protocol implementation.
|
|
92
|
+
def _endpoint_options
|
|
93
|
+
case protocol
|
|
94
|
+
when :http1
|
|
95
|
+
{ protocol: ::Async::HTTP::Protocol::HTTP1 }
|
|
96
|
+
when :http2
|
|
97
|
+
{ protocol: ::Async::HTTP::Protocol::HTTP2 }
|
|
98
|
+
when nil
|
|
99
|
+
{}
|
|
100
|
+
else
|
|
101
|
+
raise ArgumentError, "Unsupported protocol: #{protocol.inspect}, expected :http1 or :http2"
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
79
105
|
def _flush!
|
|
80
106
|
queue = @queue
|
|
81
107
|
@queue = []
|
|
82
|
-
queue.each { |block|
|
|
108
|
+
queue.each { |delay, block| _queue(delay: delay, &block) }
|
|
83
109
|
end
|
|
84
110
|
|
|
85
111
|
def _close_clients!
|
|
@@ -9,6 +9,8 @@ module LinkChecker
|
|
|
9
9
|
ATTRIBUTES = %i[
|
|
10
10
|
read_timeout
|
|
11
11
|
open_timeout
|
|
12
|
+
protocol
|
|
13
|
+
max_concurrency
|
|
12
14
|
].freeze
|
|
13
15
|
|
|
14
16
|
attr_accessor(*Config::ATTRIBUTES)
|
|
@@ -16,6 +18,8 @@ module LinkChecker
|
|
|
16
18
|
def reset
|
|
17
19
|
self.read_timeout = nil
|
|
18
20
|
self.open_timeout = nil
|
|
21
|
+
self.protocol = nil
|
|
22
|
+
self.max_concurrency = nil
|
|
19
23
|
end
|
|
20
24
|
end
|
|
21
25
|
|
|
File without changes
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'json'
|
|
6
|
+
require 'tempfile'
|
|
7
|
+
require 'uri'
|
|
8
|
+
|
|
9
|
+
module LinkChecker
|
|
10
|
+
class CachedResult < Result
|
|
11
|
+
attr_reader :code
|
|
12
|
+
|
|
13
|
+
def initialize(data, uri)
|
|
14
|
+
@code = data.fetch('code')
|
|
15
|
+
super(URI(data.fetch('result_uri')), data.fetch('method'), URI(uri.to_s))
|
|
16
|
+
@cached = true
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def success?
|
|
20
|
+
true
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
class Cache
|
|
25
|
+
VERSION = 1
|
|
26
|
+
|
|
27
|
+
def initialize(options, logger:)
|
|
28
|
+
unless options.is_a?(Hash) && options[:file].is_a?(String) && !options[:file].strip.empty?
|
|
29
|
+
raise ArgumentError, 'Cache requires a file path'
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
@ttl = options[:ttl]
|
|
33
|
+
unless @ttl.is_a?(Numeric) && @ttl.real? && @ttl.finite? && @ttl.positive?
|
|
34
|
+
raise ArgumentError, 'Cache requires a positive, finite TTL in seconds'
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
@file = File.expand_path(options[:file])
|
|
38
|
+
@logger = logger
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def fetch(key, uri)
|
|
42
|
+
data = read_entries[key]
|
|
43
|
+
return unless fresh?(data) && data['code'].is_a?(Integer) && data['code'].between?(200, 299)
|
|
44
|
+
return unless data['method'].is_a?(String) && data['result_uri'].is_a?(String)
|
|
45
|
+
|
|
46
|
+
parsed = URI(data['result_uri'])
|
|
47
|
+
return unless %w[http https].include?(parsed.scheme) && parsed.host && !parsed.userinfo
|
|
48
|
+
|
|
49
|
+
CachedResult.new(data, uri)
|
|
50
|
+
rescue URI::InvalidURIError
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def store(key, result)
|
|
55
|
+
return unless result.success?
|
|
56
|
+
return if URI(result.result_uri.to_s).userinfo
|
|
57
|
+
|
|
58
|
+
directory = File.dirname(@file)
|
|
59
|
+
FileUtils.mkdir_p(directory)
|
|
60
|
+
File.open("#{@file}.lock", File::RDWR | File::CREAT, 0o600) do |lock|
|
|
61
|
+
lock.flock(File::LOCK_EX)
|
|
62
|
+
entries = read_entries.select { |_entry_key, data| fresh?(data) }
|
|
63
|
+
entries[key] = {
|
|
64
|
+
'checked_at' => Time.now.to_f,
|
|
65
|
+
'method' => result.method,
|
|
66
|
+
'code' => result.code,
|
|
67
|
+
'result_uri' => result.result_uri.to_s
|
|
68
|
+
}
|
|
69
|
+
Tempfile.create(['link-checker-', '.json'], directory) do |file|
|
|
70
|
+
file.write(JSON.generate('version' => VERSION, 'entries' => entries))
|
|
71
|
+
file.flush
|
|
72
|
+
File.rename(file.path, @file)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
rescue IOError, SystemCallError, JSON::GeneratorError
|
|
76
|
+
@logger.warn("Unable to write link cache #{@file}; result was not cached")
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
def fresh?(data)
|
|
82
|
+
return false unless data.is_a?(Hash)
|
|
83
|
+
|
|
84
|
+
timestamp = data['checked_at']
|
|
85
|
+
now = Time.now.to_f
|
|
86
|
+
timestamp.is_a?(Numeric) && timestamp.finite? && timestamp <= now && now - timestamp < @ttl
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def read_entries
|
|
90
|
+
data = JSON.parse(File.read(@file))
|
|
91
|
+
return {} unless data.is_a?(Hash) && data['version'] == VERSION && data['entries'].is_a?(Hash)
|
|
92
|
+
|
|
93
|
+
data['entries']
|
|
94
|
+
rescue Errno::ENOENT
|
|
95
|
+
{}
|
|
96
|
+
rescue IOError, SystemCallError, JSON::ParserError
|
|
97
|
+
@logger.warn("Unable to read link cache #{@file}; checking links normally")
|
|
98
|
+
{}
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
File without changes
|
|
@@ -6,15 +6,17 @@ module LinkChecker
|
|
|
6
6
|
include LinkChecker::Callbacks
|
|
7
7
|
|
|
8
8
|
attr_reader :results
|
|
9
|
-
attr_accessor(*Config::ATTRIBUTES)
|
|
10
9
|
|
|
11
10
|
def initialize(options = {})
|
|
12
11
|
LinkChecker::Config::ATTRIBUTES.each do |key|
|
|
13
|
-
send("#{key}=", options
|
|
12
|
+
send("#{key}=", options.fetch(key) { LinkChecker.config.send(key) })
|
|
14
13
|
end
|
|
15
14
|
raise ArgumentError, "Missing methods." if methods&.none?
|
|
16
15
|
@logger ||= options[:logger] || LinkChecker::Config.logger || LinkChecker::Logger.default
|
|
17
16
|
@results = { error: [], failure: [], success: [] } unless options.key?(:results) && !options[:results]
|
|
17
|
+
@pending = {}
|
|
18
|
+
@successful = {}
|
|
19
|
+
@cache_store = Cache.new(cache, logger: @logger) if cache
|
|
18
20
|
end
|
|
19
21
|
|
|
20
22
|
def task_klass
|
|
@@ -25,6 +27,37 @@ module LinkChecker
|
|
|
25
27
|
end
|
|
26
28
|
|
|
27
29
|
def check(uri, options = {})
|
|
30
|
+
return execute_check(uri, options) unless deduplicate || @cache_store
|
|
31
|
+
|
|
32
|
+
key = request_key(uri)
|
|
33
|
+
return execute_check(uri, options) unless key
|
|
34
|
+
|
|
35
|
+
subscriber = [uri.to_s.dup, options]
|
|
36
|
+
if deduplicate && (entry = @pending[key])
|
|
37
|
+
entry[:subscribers] << subscriber
|
|
38
|
+
entry[:events].each { |event, result| publish(event, copy_result(result, subscriber)) }
|
|
39
|
+
return
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
cached_result = @successful[key] if deduplicate
|
|
43
|
+
cached_result ||= @cache_store&.fetch(key, uri)
|
|
44
|
+
if cached_result
|
|
45
|
+
@successful[key] ||= cached_result if deduplicate
|
|
46
|
+
result = copy_result(cached_result, subscriber)
|
|
47
|
+
result.cached = true
|
|
48
|
+
publish(:result, result)
|
|
49
|
+
publish(:success, result)
|
|
50
|
+
return
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
entry = { subscribers: [subscriber], events: [] }
|
|
54
|
+
@pending[key] = entry if deduplicate
|
|
55
|
+
execute_check(uri, options) { |event, result| handle_shared_result(key, entry, uri, event, result) }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def execute_check(uri, options, &handler)
|
|
28
61
|
tasks = Tasks.new(
|
|
29
62
|
self,
|
|
30
63
|
task_klass,
|
|
@@ -33,10 +66,49 @@ module LinkChecker
|
|
|
33
66
|
options
|
|
34
67
|
)
|
|
35
68
|
tasks.on do |event, *args|
|
|
36
|
-
|
|
37
|
-
callback event, *args
|
|
69
|
+
handler ? handler.call(event, *args) : publish(event, args.first)
|
|
38
70
|
end
|
|
39
71
|
tasks.execute!
|
|
40
72
|
end
|
|
73
|
+
|
|
74
|
+
def publish(event, result)
|
|
75
|
+
result.checker = self
|
|
76
|
+
results[event] << result if @results && %i[error failure success].include?(event)
|
|
77
|
+
callback(event, result)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def handle_shared_result(key, entry, uri, event, result)
|
|
81
|
+
entry[:events] << [event, result]
|
|
82
|
+
subscribers = entry[:subscribers].dup
|
|
83
|
+
if %i[error failure success].include?(event)
|
|
84
|
+
@pending.delete(key) if @pending[key].equal?(entry)
|
|
85
|
+
if event == :success && request_key(uri) == key
|
|
86
|
+
@successful[key] = result if deduplicate
|
|
87
|
+
@cache_store&.store(key, result)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
subscribers.each { |subscriber| publish(event, copy_result(result, subscriber)) }
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def copy_result(result, subscriber)
|
|
94
|
+
result.dup.tap do |copy|
|
|
95
|
+
copy.uri = URI(subscriber.first)
|
|
96
|
+
copy.result_uri = result.result_uri.dup
|
|
97
|
+
copy.options = subscriber.last
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def request_key(uri)
|
|
102
|
+
parsed = URI(uri.to_s)
|
|
103
|
+
return unless %w[http https].include?(parsed.scheme) && parsed.host && !parsed.userinfo
|
|
104
|
+
|
|
105
|
+
attributes = Config::ATTRIBUTES - %i[logger cache deduplicate]
|
|
106
|
+
attributes += %i[timeout connecttimeout read_timeout open_timeout protocol].select { |key| respond_to?(key) }
|
|
107
|
+
settings = attributes.to_h { |key| [key, send(key)] }
|
|
108
|
+
settings[:headers] = headers.sort_by { |key, _value| key.to_s }
|
|
109
|
+
Digest::SHA256.hexdigest(JSON.generate([task_klass.name, parsed.to_s, settings]))
|
|
110
|
+
rescue URI::InvalidURIError, JSON::GeneratorError, TypeError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
41
113
|
end
|
|
42
114
|
end
|
|
@@ -10,6 +10,15 @@ module LinkChecker
|
|
|
10
10
|
headers
|
|
11
11
|
logger
|
|
12
12
|
retries
|
|
13
|
+
max_redirects
|
|
14
|
+
retry_status_codes
|
|
15
|
+
retry_after
|
|
16
|
+
retry_backoff
|
|
17
|
+
retry_jitter
|
|
18
|
+
max_retry_delay
|
|
19
|
+
respect_retry_after
|
|
20
|
+
deduplicate
|
|
21
|
+
cache
|
|
13
22
|
].freeze
|
|
14
23
|
|
|
15
24
|
attr_accessor(*Config::ATTRIBUTES)
|
|
@@ -20,6 +29,15 @@ module LinkChecker
|
|
|
20
29
|
self.headers = {}
|
|
21
30
|
self.logger = nil
|
|
22
31
|
self.retries = 0
|
|
32
|
+
self.max_redirects = 20
|
|
33
|
+
self.retry_status_codes = [408, 425, 429, 500, 502, 503, 504]
|
|
34
|
+
self.retry_after = 0
|
|
35
|
+
self.retry_backoff = 1
|
|
36
|
+
self.retry_jitter = 0
|
|
37
|
+
self.max_retry_delay = 60
|
|
38
|
+
self.respect_retry_after = false
|
|
39
|
+
self.deduplicate = false
|
|
40
|
+
self.cache = nil
|
|
23
41
|
end
|
|
24
42
|
|
|
25
43
|
def retries=(value)
|
|
@@ -27,6 +45,50 @@ module LinkChecker
|
|
|
27
45
|
|
|
28
46
|
@retries = value
|
|
29
47
|
end
|
|
48
|
+
|
|
49
|
+
def max_redirects=(value)
|
|
50
|
+
raise ArgumentError, "Invalid redirect limit: #{value}" unless value.is_a?(Integer) && value >= 0
|
|
51
|
+
|
|
52
|
+
@max_redirects = value
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def retry_status_codes=(value)
|
|
56
|
+
unless value.nil? || (value.is_a?(Array) && value.all? { |code| code.is_a?(Integer) && code.between?(100, 599) })
|
|
57
|
+
raise ArgumentError, "Invalid retry status codes: #{value}"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
@retry_status_codes = value&.dup
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def retry_after=(value)
|
|
64
|
+
validate_number!(:retry_after, value, 0)
|
|
65
|
+
@retry_after = value
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def retry_backoff=(value)
|
|
69
|
+
validate_number!(:retry_backoff, value, 1)
|
|
70
|
+
@retry_backoff = value
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def retry_jitter=(value)
|
|
74
|
+
validate_number!(:retry_jitter, value, 0)
|
|
75
|
+
raise ArgumentError, "Invalid retry_jitter: #{value}" if value > 1
|
|
76
|
+
|
|
77
|
+
@retry_jitter = value
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def max_retry_delay=(value)
|
|
81
|
+
validate_number!(:max_retry_delay, value, 0)
|
|
82
|
+
@max_retry_delay = value
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def validate_number!(name, value, minimum)
|
|
88
|
+
return if value.is_a?(Numeric) && value.real? && value.finite? && value >= minimum
|
|
89
|
+
|
|
90
|
+
raise ArgumentError, "Invalid #{name}: #{value}"
|
|
91
|
+
end
|
|
30
92
|
end
|
|
31
93
|
|
|
32
94
|
class << self
|
|
File without changes
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LinkChecker
|
|
4
|
+
module Errors
|
|
5
|
+
class ConnectionError < BaseError
|
|
6
|
+
TRANSIENT_CODES = %i[
|
|
7
|
+
couldnt_resolve_proxy couldnt_resolve_host couldnt_connect
|
|
8
|
+
partial_file got_nothing send_error recv_error http2 http2_stream
|
|
9
|
+
].freeze
|
|
10
|
+
|
|
11
|
+
attr_reader :return_code
|
|
12
|
+
|
|
13
|
+
def initialize(return_code, message)
|
|
14
|
+
@return_code = return_code
|
|
15
|
+
super(message)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def error_type
|
|
19
|
+
TRANSIENT_CODES.include?(return_code) ? :connection : :other
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LinkChecker
|
|
4
|
+
module Errors
|
|
5
|
+
class RedirectLimitError < BaseError
|
|
6
|
+
attr_reader :limit, :urls
|
|
7
|
+
|
|
8
|
+
def initialize(limit, urls)
|
|
9
|
+
@limit = limit
|
|
10
|
+
@urls = urls
|
|
11
|
+
super("Redirect limit of #{limit} exceeded: #{urls.join(' -> ')}.")
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|