ruby-link-checker 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/coverage.yml +24 -0
  3. data/.github/workflows/danger-comment.yml +17 -0
  4. data/.github/workflows/danger.yml +12 -0
  5. data/.github/workflows/lint.yml +20 -0
  6. data/.github/workflows/release.yml +14 -0
  7. data/.github/workflows/test.yml +25 -0
  8. data/.gitignore +6 -0
  9. data/.rspec +2 -0
  10. data/.rubocop.yml +26 -0
  11. data/.rubocop_todo.yml +332 -0
  12. data/AGENTS.md +23 -0
  13. data/CHANGELOG.md +15 -0
  14. data/CONTRIBUTING.md +4 -0
  15. data/Dangerfile +0 -0
  16. data/Gemfile +3 -0
  17. data/LICENSE.md +0 -0
  18. data/README.md +78 -4
  19. data/RELEASING.md +17 -40
  20. data/Rakefile +8 -0
  21. data/benchmarks/check.rb +0 -0
  22. data/benchmarks/data/opensearch.org/links.txt +0 -0
  23. data/benchmarks/data/opensearch.org/small.txt +0 -0
  24. data/lib/ruby-link-checker/async/http/checker.rb +32 -6
  25. data/lib/ruby-link-checker/async/http/config.rb +4 -0
  26. data/lib/ruby-link-checker/async/http/result.rb +4 -0
  27. data/lib/ruby-link-checker/async/http.rb +0 -0
  28. data/lib/ruby-link-checker/cache.rb +101 -0
  29. data/lib/ruby-link-checker/callbacks.rb +0 -0
  30. data/lib/ruby-link-checker/checker.rb +76 -4
  31. data/lib/ruby-link-checker/config.rb +62 -0
  32. data/lib/ruby-link-checker/errors/base_error.rb +0 -0
  33. data/lib/ruby-link-checker/errors/connection_error.rb +23 -0
  34. data/lib/ruby-link-checker/errors/redirect_limit_error.rb +15 -0
  35. data/lib/ruby-link-checker/errors/redirect_loop_error.rb +0 -0
  36. data/lib/ruby-link-checker/errors.rb +2 -0
  37. data/lib/ruby-link-checker/logger.rb +0 -0
  38. data/lib/ruby-link-checker/net/http/checker.rb +2 -1
  39. data/lib/ruby-link-checker/net/http/config.rb +0 -0
  40. data/lib/ruby-link-checker/net/http/result.rb +4 -0
  41. data/lib/ruby-link-checker/net/http.rb +0 -0
  42. data/lib/ruby-link-checker/result.rb +25 -0
  43. data/lib/ruby-link-checker/task.rb +2 -0
  44. data/lib/ruby-link-checker/tasks.rb +40 -6
  45. data/lib/ruby-link-checker/typhoeus/hydra/checker.rb +27 -5
  46. data/lib/ruby-link-checker/typhoeus/hydra/config.rb +0 -0
  47. data/lib/ruby-link-checker/typhoeus/hydra/result.rb +4 -0
  48. data/lib/ruby-link-checker/typhoeus/hydra.rb +0 -0
  49. data/lib/ruby-link-checker/version.rb +1 -1
  50. data/lib/ruby-link-checker.rb +1 -0
  51. data/lib/ruby_link_checker.rb +0 -0
  52. data/ruby-link-checker.gemspec +1 -1
  53. data/spec/fixtures/200.yml +0 -0
  54. data/spec/fixtures/301_200.yml +0 -0
  55. data/spec/fixtures/301_301.yml +0 -0
  56. data/spec/fixtures/301_400_301_200.yml +0 -0
  57. data/spec/fixtures/301_403.yml +0 -0
  58. data/spec/fixtures/308_429.yml +0 -0
  59. data/spec/fixtures/404.yml +0 -0
  60. data/spec/fixtures/404_200.yml +0 -0
  61. data/spec/fixtures/404_404.yml +0 -0
  62. data/spec/fixtures/429_200.yml +0 -0
  63. data/spec/fixtures/429_429_200.yml +0 -0
  64. data/spec/ruby-link-checker/async/http/checker_spec.rb +86 -1
  65. data/spec/ruby-link-checker/callbacks_spec.rb +0 -0
  66. data/spec/ruby-link-checker/checker_spec.rb +89 -2
  67. data/spec/ruby-link-checker/config_spec.rb +6 -0
  68. data/spec/ruby-link-checker/errors/redirect_loop_error_spec.rb +0 -0
  69. data/spec/ruby-link-checker/net/http/checker_spec.rb +15 -0
  70. data/spec/ruby-link-checker/result_error_spec.rb +17 -0
  71. data/spec/ruby-link-checker/result_spec.rb +0 -0
  72. data/spec/ruby-link-checker/task_spec.rb +0 -0
  73. data/spec/ruby-link-checker/tasks_spec.rb +44 -0
  74. data/spec/ruby-link-checker/typhoeus/hydra/checker_spec.rb +88 -1
  75. data/spec/ruby-link-checker/version_spec.rb +0 -0
  76. data/spec/spec_helper.rb +0 -0
  77. data/spec/support/config.rb +0 -0
  78. data/spec/support/link_checker.rb +243 -3
  79. data/spec/support/vcr.rb +0 -0
  80. data/spec/support/with_result.rb +0 -0
  81. data/spec/support/with_url.rb +0 -0
  82. metadata +21 -34
  83. data/Gemfile.lock +0 -334
  84. data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_asc.png +0 -0
  85. data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_asc_disabled.png +0 -0
  86. data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_both.png +0 -0
  87. data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_desc.png +0 -0
  88. data/coverage/assets/0.13.2/DataTables-1.10.20/images/sort_desc_disabled.png +0 -0
  89. data/coverage/assets/0.13.2/application.css +0 -1
  90. data/coverage/assets/0.13.2/application.js +0 -7
  91. data/coverage/assets/0.13.2/colorbox/border.png +0 -0
  92. data/coverage/assets/0.13.2/colorbox/controls.png +0 -0
  93. data/coverage/assets/0.13.2/colorbox/loading.gif +0 -0
  94. data/coverage/assets/0.13.2/colorbox/loading_background.png +0 -0
  95. data/coverage/assets/0.13.2/favicon_green.png +0 -0
  96. data/coverage/assets/0.13.2/favicon_red.png +0 -0
  97. data/coverage/assets/0.13.2/favicon_yellow.png +0 -0
  98. data/coverage/assets/0.13.2/images/ui-bg_flat_0_aaaaaa_40x100.png +0 -0
  99. data/coverage/assets/0.13.2/images/ui-bg_flat_75_ffffff_40x100.png +0 -0
  100. data/coverage/assets/0.13.2/images/ui-bg_glass_55_fbf9ee_1x400.png +0 -0
  101. data/coverage/assets/0.13.2/images/ui-bg_glass_65_ffffff_1x400.png +0 -0
  102. data/coverage/assets/0.13.2/images/ui-bg_glass_75_dadada_1x400.png +0 -0
  103. data/coverage/assets/0.13.2/images/ui-bg_glass_75_e6e6e6_1x400.png +0 -0
  104. data/coverage/assets/0.13.2/images/ui-bg_glass_95_fef1ec_1x400.png +0 -0
  105. data/coverage/assets/0.13.2/images/ui-bg_highlight-soft_75_cccccc_1x100.png +0 -0
  106. data/coverage/assets/0.13.2/images/ui-icons_222222_256x240.png +0 -0
  107. data/coverage/assets/0.13.2/images/ui-icons_2e83ff_256x240.png +0 -0
  108. data/coverage/assets/0.13.2/images/ui-icons_454545_256x240.png +0 -0
  109. data/coverage/assets/0.13.2/images/ui-icons_888888_256x240.png +0 -0
  110. data/coverage/assets/0.13.2/images/ui-icons_cd0a0a_256x240.png +0 -0
  111. data/coverage/assets/0.13.2/loading.gif +0 -0
  112. data/coverage/assets/0.13.2/magnify.png +0 -0
  113. data/coverage/index.html +0 -10746
data/README.md CHANGED
@@ -19,9 +19,13 @@ A fast Ruby link checker with support for multiple HTTP libraries. Does not pars
19
19
  - [LinkChecker::Net::HTTP](#linkcheckernethttp)
20
20
  - [Options](#options)
21
21
  - [Retries](#retries)
22
+ - [Redirects](#redirects)
22
23
  - [Retry After](#retry-after)
24
+ - [Retry Policy](#retry-policy)
23
25
  - [Results](#results)
24
26
  - [Methods](#methods)
27
+ - [Deduplication](#deduplication)
28
+ - [Caching](#caching)
25
29
  - [Logger](#logger)
26
30
  - [User-Agent](#user-agent)
27
31
  - [Headers](#headers)
@@ -172,6 +176,20 @@ LinkChecker::Async::HTTP.configure do |config|
172
176
  end
173
177
  ```
174
178
 
179
+ You can also force a specific HTTP protocol version, `:http1` or `:http2`, per instance or globally. By default, `async-http` negotiates the protocol version with the server.
180
+
181
+ ```ruby
182
+ LinkChecker::Async::HTTP.configure do |config|
183
+ config.protocol = :http2
184
+ end
185
+ ```
186
+
187
+ Async::HTTP reuses connection-pooling clients per scheme/host/port. Its default concurrency is unlimited; set `max_concurrency` to bound active requests without counting delayed retries against that limit.
188
+
189
+ ```ruby
190
+ checker = LinkChecker::Async::HTTP::Checker.new(max_concurrency: 25)
191
+ ```
192
+
175
193
  #### [LinkChecker::Net::HTTP](lib/ruby-link-checker/net/http/checker.rb)
176
194
 
177
195
  Slow, sequential checker.
@@ -210,18 +228,43 @@ end
210
228
 
211
229
  #### Retries
212
230
 
213
- By default link checkers do not retry. You can set a number of times to retry all errors and failures with `retries`.
231
+ By default link checkers do not retry. Set `retries` to enable retries for transient connection errors/timeouts and HTTP statuses 408, 425, 429, 500, 502, 503, and 504. Permanent HTTP failures and invalid URLs are not retried. HEAD-to-GET fallback still applies independently of the retry policy. The retry budget is shared across methods: with `retries: 2`, repeated transient failures produce `HEAD`, `HEAD`, `HEAD`, then `GET`.
214
232
 
215
233
  ```ruby
216
- checker = LinkChecker::Net::HTTP::Checker.new(retry: 1)
234
+ checker = LinkChecker::Net::HTTP::Checker.new(retries: 1)
235
+ ```
236
+
237
+ #### Redirects
238
+
239
+ Checkers follow up to 20 redirects per attempt and detect redirect loops. Relative redirects are resolved against the current response URL. Set `max_redirects` to change the limit, or use `0` to disable redirects.
240
+
241
+ ```ruby
242
+ checker = LinkChecker::Typhoeus::Hydra::Checker.new(max_redirects: 10)
217
243
  ```
218
244
 
219
245
  #### Retry After
220
246
 
221
- If retry is enabled, link checkers can wait for a number of seconds before retrying.
247
+ Set `retry_after` to a minimum delay in seconds (default `0`). Optional `retry_backoff` multiplies that delay on successive retries (default `1`), and `retry_jitter` adds a random fraction between `0` and the configured value (default `0`, maximum `1`). All delays are capped by `max_retry_delay` (default `60` seconds).
248
+
249
+ ```ruby
250
+ checker = LinkChecker::Typhoeus::Hydra::Checker.new(
251
+ retries: 2,
252
+ retry_after: 1,
253
+ retry_backoff: 2,
254
+ retry_jitter: 0.1,
255
+ respect_retry_after: true,
256
+ max_retry_delay: 60
257
+ )
258
+ ```
259
+
260
+ With `respect_retry_after: true` (default `false`), HTTP `Retry-After` delta-seconds and dates provide a minimum delay, subject to the same cap. Invalid or past dates are ignored. Async delays yield without blocking other requests. Hydra queues delayed retries after the active batch drains, so actual delays can exceed the configured minimum without blocking completion callbacks. Delays apply to retries, not ordinary HEAD-to-GET fallback.
261
+
262
+ #### Retry Policy
263
+
264
+ Customize `retry_status_codes` to choose retryable HTTP statuses. Set it to `nil` to restore the previous behavior of retrying every failure and error, including malformed URLs and programming errors.
222
265
 
223
266
  ```ruby
224
- checker = LinkChecker::Net::HTTP::Checker.new(retry_after: 1) # wait for 1 second before retrying
267
+ checker = LinkChecker::Net::HTTP::Checker.new(retries: 1, retry_status_codes: [429, 503])
225
268
  ```
226
269
 
227
270
  #### Results
@@ -256,6 +299,34 @@ The following examples disables `GET` and only makes `HEAD` requests.
256
299
  checker = LinkChecker::Net::HTTP::Checker.new(methods: %w[HEAD])
257
300
  ```
258
301
 
302
+ #### Deduplication
303
+
304
+ Set `deduplicate: true` to coalesce identical in-flight requests and reuse successful results for the lifetime of the checker. Failures and errors are rechecked after completion. Each submission still receives its own result events and metadata, including events emitted before it joined an in-flight check. Deduplication is disabled by default.
305
+
306
+ ```ruby
307
+ checker = LinkChecker::Typhoeus::Hydra::Checker.new(deduplicate: true)
308
+ checker.check('https://www.example.org', location: 'one.html')
309
+ checker.check('https://www.example.org', location: 'two.html')
310
+ checker.run # one check, two sets of result events
311
+ ```
312
+
313
+ Identity includes the complete URL (including query values and fragment), transport, methods, headers, user-agent, and request-related settings. Changing those settings prevents reuse. URLs containing user-info credentials bypass deduplication and caching.
314
+
315
+ #### Caching
316
+
317
+ Persistent success caching is disabled by default. Enable it with an explicit file path and positive TTL in seconds. Recent successful checks can then be reused across checker instances; failures and errors are never stored. Cache writes are atomic and locked, and unreadable or malformed files fall back to normal checking.
318
+
319
+ ```ruby
320
+ checker = LinkChecker::Typhoeus::Hydra::Checker.new(
321
+ deduplicate: true,
322
+ cache: { file: 'tmp/link-checker/cache.json', ttl: 3600 }
323
+ )
324
+ ```
325
+
326
+ Each cache hit emits `:result` and `:success` with the new submission's metadata and `cached? == true`. Persistent hits return `LinkChecker::CachedResult` with the HTTP method, status, original/final URIs, and metadata; native request/response objects and headers are not persisted. Redirect targets containing user-info credentials are not written. Add the cache directory to your ignore rules.
327
+
328
+ Caching alone does not coalesce in-flight checks. With deduplication enabled, successful results remain reusable within that checker even after the persistent TTL expires. Cached successes do not replay historical redirect/retry events.
329
+
259
330
  #### Logger
260
331
 
261
332
  Pass your own logger.
@@ -334,6 +405,9 @@ Events are called with results, which contain the following properties.
334
405
  | :request_headers | Request headers. |
335
406
  | :redirect_to | A redirect URL in case of redirects. |
336
407
  | :error | A raised error in case of errors. |
408
+ | :error_type | `:timeout`, `:connection`, or `:other` for errors; otherwise `nil`. |
409
+ | :response_headers | Response headers, accessible using `[]`. |
410
+ | :cached? | Whether this result was reused from a successful check. |
337
411
 
338
412
  See [result.rb](lib/ruby-link-checker/result.rb) for more details.
339
413
 
data/RELEASING.md CHANGED
@@ -2,59 +2,36 @@
2
2
 
3
3
  There are no hard rules about when to release ruby-link-checker. Release bug fixes frequently, features not so frequently and breaking API changes rarely.
4
4
 
5
- ### Release
5
+ Releases use [ruby-gem-release](https://github.com/dblock/ruby-gem-release/tree/v0.3.1) and the manual [Release workflow](.github/workflows/release.yml). Merges do not publish. Every workflow dispatch performs a real release.
6
6
 
7
- Run tests, check that all tests succeed locally.
7
+ ## One-Time Setup
8
8
 
9
- ```
10
- bundle install
11
- rake
12
- ```
13
-
14
- Check that the last build succeeded.
15
-
16
- Add a date to this release in [CHANGELOG.md](CHANGELOG.md).
17
-
18
- ```
19
- ### 0.2.2 (2015/7/10)
20
- ```
9
+ Create a GitHub environment named `release` in this repository. Restrict deployment to `main` and require approval when available.
21
10
 
22
- Remove the line with "Your contribution here.", since there will be no more contributions to this release.
11
+ On [RubyGems.org](https://rubygems.org/gems/ruby-link-checker/trusted_publishers), configure a trusted publisher with repository owner `dblock`, repository name `ruby-link-checker`, workflow filename `release.yml`, and environment `release`. Set the separate workflow repository owner to `dblock` and name to `ruby-gem-release` because publication uses its reusable workflow. No stored RubyGems API key is needed.
23
12
 
24
- Commit your changes.
13
+ The workflow requests `contents: write` and `id-token: write`. Branch rules must allow the release identity to push fast-forward release and next-version commits to `main` and create tags, without allowing force pushes or deletion. Adding the workflow does not configure these permissions or the trusted publisher.
25
14
 
26
- ```
27
- git add README.md CHANGELOG.md lib/ruby-link-checker/version.rb
28
- git commit -m "Preparing for release, 0.2.2."
29
- ```
15
+ ## Prepare and Publish
30
16
 
31
- Release.
17
+ Prepare the intended version in `lib/ruby-link-checker/version.rb` and its matching `### VERSION (Next)` section in `CHANGELOG.md` through a PR. Keep exactly one `* Your contribution here.` placeholder in that section. The workflow releases the current version, dates the changelog, and removes the placeholder; it does not select a new release version.
32
18
 
33
- ```
34
- $ rake release
19
+ Run tests and RuboCop locally and review the release changes before merging:
35
20
 
36
- ruby-link-checker 0.2.2 built to pkg/ruby-link-checker-0.2.2.gem.
37
- Tagged v0.2.2.
38
- Pushed git commits and tags.
39
- Pushed ruby-link-checker 0.2.2 to rubygems.org.
21
+ ```sh
22
+ bundle exec rake
40
23
  ```
41
24
 
42
- ### Prepare for the Next Version
43
-
44
- Add the next release to [CHANGELOG.md](CHANGELOG.md).
25
+ After merging, explicit release approval, and successful Trusted Publishing setup, dispatch from `main`:
45
26
 
27
+ ```sh
28
+ gh workflow run release.yml --ref main
46
29
  ```
47
- ### 0.2.3 (Next)
48
30
 
49
- * Your contribution here.
50
- ```
31
+ The workflow runs `bundle exec rake`, publishes the verified gem to RubyGems, pushes the release commit and tag atomically, and creates a GitHub Release with the gem attached. It then increments the patch version and pushes a new development commit with a fresh changelog section. Actions-token pushes do not trigger ordinary push workflows, so the release workflow runs tests and RuboCop before publication.
51
32
 
52
- Increment the third version number in [lib/ruby-link-checker/version.rb](lib/ruby-link-checker/version.rb).
33
+ ## Verify and Recover
53
34
 
54
- Commit your changes.
35
+ Verify the published RubyGems version, matching tag and GitHub Release, attached gem, and next development version on `main`.
55
36
 
56
- ```
57
- git add CHANGELOG.md lib/ruby-link-checker/version.rb
58
- git commit -m "Preparing for next development iteration, 0.2.3."
59
- git push origin main
60
- ```
37
+ Never automatically rerun a failed or timed-out publication: RubyGems may have accepted the upload. Check registry and GitHub state first. Do not republish an existing version or move its tag. Available `release.gem`, `release.json`, and `release.bundle` recovery artifacts are retained for 30 days; follow the [upstream recovery instructions](https://github.com/dblock/ruby-gem-release/blob/v0.3.1/RELEASING.md#verify-and-recover) to complete only the missing steps.
data/Rakefile CHANGED
@@ -15,4 +15,12 @@ end
15
15
  require 'rubocop/rake_task'
16
16
  RuboCop::RakeTask.new(:rubocop)
17
17
 
18
+ require 'ruby-gem-release/workflow/ruby_gem'
19
+ require 'ruby-gem-release/changelog'
20
+
21
+ RubyGemRelease::Workflow::RubyGem.new(
22
+ version: RubyGemRelease::Version::RubyConstant.new(path: 'lib/ruby-link-checker/version.rb', constant: 'VERSION'),
23
+ changelog: RubyGemRelease::Changelog::Intridea.new(placeholder: '* Your contribution here.')
24
+ ).install_tasks
25
+
18
26
  task default: %i[rubocop spec]
data/benchmarks/check.rb CHANGED
File without changes
File without changes
File without changes
@@ -3,7 +3,7 @@ module LinkChecker
3
3
  module HTTP
4
4
  class Task < ::LinkChecker::Task
5
5
  def run!
6
- checker._queue { _run! }
6
+ checker._queue(delay: delay) { _run! }
7
7
  end
8
8
 
9
9
  private
@@ -42,25 +42,36 @@ module LinkChecker
42
42
  end
43
43
  @queue = []
44
44
  @clients = {}
45
+ if max_concurrency && (!max_concurrency.is_a?(Integer) || max_concurrency < 1)
46
+ raise ArgumentError, "Invalid concurrency limit: #{max_concurrency}"
47
+ end
48
+
45
49
  super
46
50
  end
47
51
 
48
52
  # Runs all queued checks concurrently and blocks until they complete.
49
53
  def run
54
+ require 'async/semaphore'
55
+
50
56
  Barrier do |barrier|
51
57
  @barrier = barrier
58
+ @semaphore = ::Async::Semaphore.new(max_concurrency) if max_concurrency
52
59
  _flush!
53
60
  end
54
61
  ensure
55
62
  @barrier = nil
63
+ @semaphore = nil
56
64
  _close_clients!
57
65
  end
58
66
 
59
- def _queue(&block)
67
+ def _queue(delay: 0, &block)
60
68
  if @barrier
61
- @barrier.async(&block)
69
+ @barrier.async do
70
+ sleep(delay) if delay.positive?
71
+ @semaphore ? @semaphore.acquire(&block) : block.call
72
+ end
62
73
  else
63
- @queue << block
74
+ @queue << [delay, block]
64
75
  end
65
76
  end
66
77
 
@@ -69,17 +80,32 @@ module LinkChecker
69
80
  def _client_for(uri)
70
81
  key = [uri.scheme, uri.host, uri.port]
71
82
  @clients[key] ||= begin
72
- endpoint = ::Async::HTTP::Endpoint.parse(uri.to_s, timeout: open_timeout)
83
+ endpoint = ::Async::HTTP::Endpoint.parse(uri.to_s, timeout: open_timeout, **_endpoint_options)
73
84
  ::Async::HTTP::Client.new(endpoint)
74
85
  end
75
86
  end
76
87
 
77
88
  private
78
89
 
90
+ # Maps the configured protocol (:http1 or :http2) to the
91
+ # corresponding async-http protocol implementation.
92
+ def _endpoint_options
93
+ case protocol
94
+ when :http1
95
+ { protocol: ::Async::HTTP::Protocol::HTTP1 }
96
+ when :http2
97
+ { protocol: ::Async::HTTP::Protocol::HTTP2 }
98
+ when nil
99
+ {}
100
+ else
101
+ raise ArgumentError, "Unsupported protocol: #{protocol.inspect}, expected :http1 or :http2"
102
+ end
103
+ end
104
+
79
105
  def _flush!
80
106
  queue = @queue
81
107
  @queue = []
82
- queue.each { |block| @barrier.async(&block) }
108
+ queue.each { |delay, block| _queue(delay: delay, &block) }
83
109
  end
84
110
 
85
111
  def _close_clients!
@@ -9,6 +9,8 @@ module LinkChecker
9
9
  ATTRIBUTES = %i[
10
10
  read_timeout
11
11
  open_timeout
12
+ protocol
13
+ max_concurrency
12
14
  ].freeze
13
15
 
14
16
  attr_accessor(*Config::ATTRIBUTES)
@@ -16,6 +18,8 @@ module LinkChecker
16
18
  def reset
17
19
  self.read_timeout = nil
18
20
  self.open_timeout = nil
21
+ self.protocol = nil
22
+ self.max_concurrency = nil
19
23
  end
20
24
  end
21
25
 
@@ -58,6 +58,10 @@ module LinkChecker
58
58
  Headers.new(request.headers)
59
59
  end
60
60
 
61
+ def response_headers
62
+ response&.headers || {}
63
+ end
64
+
61
65
  def redirect_to
62
66
  return nil unless response
63
67
 
File without changes
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+ require 'fileutils'
5
+ require 'json'
6
+ require 'tempfile'
7
+ require 'uri'
8
+
9
+ module LinkChecker
10
+ class CachedResult < Result
11
+ attr_reader :code
12
+
13
+ def initialize(data, uri)
14
+ @code = data.fetch('code')
15
+ super(URI(data.fetch('result_uri')), data.fetch('method'), URI(uri.to_s))
16
+ @cached = true
17
+ end
18
+
19
+ def success?
20
+ true
21
+ end
22
+ end
23
+
24
+ class Cache
25
+ VERSION = 1
26
+
27
+ def initialize(options, logger:)
28
+ unless options.is_a?(Hash) && options[:file].is_a?(String) && !options[:file].strip.empty?
29
+ raise ArgumentError, 'Cache requires a file path'
30
+ end
31
+
32
+ @ttl = options[:ttl]
33
+ unless @ttl.is_a?(Numeric) && @ttl.real? && @ttl.finite? && @ttl.positive?
34
+ raise ArgumentError, 'Cache requires a positive, finite TTL in seconds'
35
+ end
36
+
37
+ @file = File.expand_path(options[:file])
38
+ @logger = logger
39
+ end
40
+
41
+ def fetch(key, uri)
42
+ data = read_entries[key]
43
+ return unless fresh?(data) && data['code'].is_a?(Integer) && data['code'].between?(200, 299)
44
+ return unless data['method'].is_a?(String) && data['result_uri'].is_a?(String)
45
+
46
+ parsed = URI(data['result_uri'])
47
+ return unless %w[http https].include?(parsed.scheme) && parsed.host && !parsed.userinfo
48
+
49
+ CachedResult.new(data, uri)
50
+ rescue URI::InvalidURIError
51
+ nil
52
+ end
53
+
54
+ def store(key, result)
55
+ return unless result.success?
56
+ return if URI(result.result_uri.to_s).userinfo
57
+
58
+ directory = File.dirname(@file)
59
+ FileUtils.mkdir_p(directory)
60
+ File.open("#{@file}.lock", File::RDWR | File::CREAT, 0o600) do |lock|
61
+ lock.flock(File::LOCK_EX)
62
+ entries = read_entries.select { |_entry_key, data| fresh?(data) }
63
+ entries[key] = {
64
+ 'checked_at' => Time.now.to_f,
65
+ 'method' => result.method,
66
+ 'code' => result.code,
67
+ 'result_uri' => result.result_uri.to_s
68
+ }
69
+ Tempfile.create(['link-checker-', '.json'], directory) do |file|
70
+ file.write(JSON.generate('version' => VERSION, 'entries' => entries))
71
+ file.flush
72
+ File.rename(file.path, @file)
73
+ end
74
+ end
75
+ rescue IOError, SystemCallError, JSON::GeneratorError
76
+ @logger.warn("Unable to write link cache #{@file}; result was not cached")
77
+ end
78
+
79
+ private
80
+
81
+ def fresh?(data)
82
+ return false unless data.is_a?(Hash)
83
+
84
+ timestamp = data['checked_at']
85
+ now = Time.now.to_f
86
+ timestamp.is_a?(Numeric) && timestamp.finite? && timestamp <= now && now - timestamp < @ttl
87
+ end
88
+
89
+ def read_entries
90
+ data = JSON.parse(File.read(@file))
91
+ return {} unless data.is_a?(Hash) && data['version'] == VERSION && data['entries'].is_a?(Hash)
92
+
93
+ data['entries']
94
+ rescue Errno::ENOENT
95
+ {}
96
+ rescue IOError, SystemCallError, JSON::ParserError
97
+ @logger.warn("Unable to read link cache #{@file}; checking links normally")
98
+ {}
99
+ end
100
+ end
101
+ end
File without changes
@@ -6,15 +6,17 @@ module LinkChecker
6
6
  include LinkChecker::Callbacks
7
7
 
8
8
  attr_reader :results
9
- attr_accessor(*Config::ATTRIBUTES)
10
9
 
11
10
  def initialize(options = {})
12
11
  LinkChecker::Config::ATTRIBUTES.each do |key|
13
- send("#{key}=", options[key] || LinkChecker.config.send(key))
12
+ send("#{key}=", options.fetch(key) { LinkChecker.config.send(key) })
14
13
  end
15
14
  raise ArgumentError, "Missing methods." if methods&.none?
16
15
  @logger ||= options[:logger] || LinkChecker::Config.logger || LinkChecker::Logger.default
17
16
  @results = { error: [], failure: [], success: [] } unless options.key?(:results) && !options[:results]
17
+ @pending = {}
18
+ @successful = {}
19
+ @cache_store = Cache.new(cache, logger: @logger) if cache
18
20
  end
19
21
 
20
22
  def task_klass
@@ -25,6 +27,37 @@ module LinkChecker
25
27
  end
26
28
 
27
29
  def check(uri, options = {})
30
+ return execute_check(uri, options) unless deduplicate || @cache_store
31
+
32
+ key = request_key(uri)
33
+ return execute_check(uri, options) unless key
34
+
35
+ subscriber = [uri.to_s.dup, options]
36
+ if deduplicate && (entry = @pending[key])
37
+ entry[:subscribers] << subscriber
38
+ entry[:events].each { |event, result| publish(event, copy_result(result, subscriber)) }
39
+ return
40
+ end
41
+
42
+ cached_result = @successful[key] if deduplicate
43
+ cached_result ||= @cache_store&.fetch(key, uri)
44
+ if cached_result
45
+ @successful[key] ||= cached_result if deduplicate
46
+ result = copy_result(cached_result, subscriber)
47
+ result.cached = true
48
+ publish(:result, result)
49
+ publish(:success, result)
50
+ return
51
+ end
52
+
53
+ entry = { subscribers: [subscriber], events: [] }
54
+ @pending[key] = entry if deduplicate
55
+ execute_check(uri, options) { |event, result| handle_shared_result(key, entry, uri, event, result) }
56
+ end
57
+
58
+ private
59
+
60
+ def execute_check(uri, options, &handler)
28
61
  tasks = Tasks.new(
29
62
  self,
30
63
  task_klass,
@@ -33,10 +66,49 @@ module LinkChecker
33
66
  options
34
67
  )
35
68
  tasks.on do |event, *args|
36
- results[event] << args.first if @results && %i[error failure success].include?(event)
37
- callback event, *args
69
+ handler ? handler.call(event, *args) : publish(event, args.first)
38
70
  end
39
71
  tasks.execute!
40
72
  end
73
+
74
+ def publish(event, result)
75
+ result.checker = self
76
+ results[event] << result if @results && %i[error failure success].include?(event)
77
+ callback(event, result)
78
+ end
79
+
80
+ def handle_shared_result(key, entry, uri, event, result)
81
+ entry[:events] << [event, result]
82
+ subscribers = entry[:subscribers].dup
83
+ if %i[error failure success].include?(event)
84
+ @pending.delete(key) if @pending[key].equal?(entry)
85
+ if event == :success && request_key(uri) == key
86
+ @successful[key] = result if deduplicate
87
+ @cache_store&.store(key, result)
88
+ end
89
+ end
90
+ subscribers.each { |subscriber| publish(event, copy_result(result, subscriber)) }
91
+ end
92
+
93
+ def copy_result(result, subscriber)
94
+ result.dup.tap do |copy|
95
+ copy.uri = URI(subscriber.first)
96
+ copy.result_uri = result.result_uri.dup
97
+ copy.options = subscriber.last
98
+ end
99
+ end
100
+
101
+ def request_key(uri)
102
+ parsed = URI(uri.to_s)
103
+ return unless %w[http https].include?(parsed.scheme) && parsed.host && !parsed.userinfo
104
+
105
+ attributes = Config::ATTRIBUTES - %i[logger cache deduplicate]
106
+ attributes += %i[timeout connecttimeout read_timeout open_timeout protocol].select { |key| respond_to?(key) }
107
+ settings = attributes.to_h { |key| [key, send(key)] }
108
+ settings[:headers] = headers.sort_by { |key, _value| key.to_s }
109
+ Digest::SHA256.hexdigest(JSON.generate([task_klass.name, parsed.to_s, settings]))
110
+ rescue URI::InvalidURIError, JSON::GeneratorError, TypeError
111
+ nil
112
+ end
41
113
  end
42
114
  end
@@ -10,6 +10,15 @@ module LinkChecker
10
10
  headers
11
11
  logger
12
12
  retries
13
+ max_redirects
14
+ retry_status_codes
15
+ retry_after
16
+ retry_backoff
17
+ retry_jitter
18
+ max_retry_delay
19
+ respect_retry_after
20
+ deduplicate
21
+ cache
13
22
  ].freeze
14
23
 
15
24
  attr_accessor(*Config::ATTRIBUTES)
@@ -20,6 +29,15 @@ module LinkChecker
20
29
  self.headers = {}
21
30
  self.logger = nil
22
31
  self.retries = 0
32
+ self.max_redirects = 20
33
+ self.retry_status_codes = [408, 425, 429, 500, 502, 503, 504]
34
+ self.retry_after = 0
35
+ self.retry_backoff = 1
36
+ self.retry_jitter = 0
37
+ self.max_retry_delay = 60
38
+ self.respect_retry_after = false
39
+ self.deduplicate = false
40
+ self.cache = nil
23
41
  end
24
42
 
25
43
  def retries=(value)
@@ -27,6 +45,50 @@ module LinkChecker
27
45
 
28
46
  @retries = value
29
47
  end
48
+
49
+ def max_redirects=(value)
50
+ raise ArgumentError, "Invalid redirect limit: #{value}" unless value.is_a?(Integer) && value >= 0
51
+
52
+ @max_redirects = value
53
+ end
54
+
55
+ def retry_status_codes=(value)
56
+ unless value.nil? || (value.is_a?(Array) && value.all? { |code| code.is_a?(Integer) && code.between?(100, 599) })
57
+ raise ArgumentError, "Invalid retry status codes: #{value}"
58
+ end
59
+
60
+ @retry_status_codes = value&.dup
61
+ end
62
+
63
+ def retry_after=(value)
64
+ validate_number!(:retry_after, value, 0)
65
+ @retry_after = value
66
+ end
67
+
68
+ def retry_backoff=(value)
69
+ validate_number!(:retry_backoff, value, 1)
70
+ @retry_backoff = value
71
+ end
72
+
73
+ def retry_jitter=(value)
74
+ validate_number!(:retry_jitter, value, 0)
75
+ raise ArgumentError, "Invalid retry_jitter: #{value}" if value > 1
76
+
77
+ @retry_jitter = value
78
+ end
79
+
80
+ def max_retry_delay=(value)
81
+ validate_number!(:max_retry_delay, value, 0)
82
+ @max_retry_delay = value
83
+ end
84
+
85
+ private
86
+
87
+ def validate_number!(name, value, minimum)
88
+ return if value.is_a?(Numeric) && value.real? && value.finite? && value >= minimum
89
+
90
+ raise ArgumentError, "Invalid #{name}: #{value}"
91
+ end
30
92
  end
31
93
 
32
94
  class << self
File without changes
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ module LinkChecker
4
+ module Errors
5
+ class ConnectionError < BaseError
6
+ TRANSIENT_CODES = %i[
7
+ couldnt_resolve_proxy couldnt_resolve_host couldnt_connect
8
+ partial_file got_nothing send_error recv_error http2 http2_stream
9
+ ].freeze
10
+
11
+ attr_reader :return_code
12
+
13
+ def initialize(return_code, message)
14
+ @return_code = return_code
15
+ super(message)
16
+ end
17
+
18
+ def error_type
19
+ TRANSIENT_CODES.include?(return_code) ? :connection : :other
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ module LinkChecker
4
+ module Errors
5
+ class RedirectLimitError < BaseError
6
+ attr_reader :limit, :urls
7
+
8
+ def initialize(limit, urls)
9
+ @limit = limit
10
+ @urls = urls
11
+ super("Redirect limit of #{limit} exceeded: #{urls.join(' -> ')}.")
12
+ end
13
+ end
14
+ end
15
+ end