ruby-link-checker 0.4.0 → 0.5.1

data/lib/ruby-link-checker/errors.rb

@@ -1,2 +1,4 @@
 require_relative 'errors/base_error'
 require_relative 'errors/redirect_loop_error'
+require_relative 'errors/redirect_limit_error'
+require_relative 'errors/connection_error'

data/lib/ruby-link-checker/net/http/checker.rb

@@ -3,7 +3,8 @@ module LinkChecker
     module HTTP
       class Task < ::LinkChecker::Task
         def run!
-          ::Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+          sleep(delay) if delay.positive?
+          ::Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
             http.read_timeout = checker.read_timeout if checker.read_timeout
             http.open_timeout = checker.open_timeout if checker.open_timeout
             request = ::Net::HTTPGenericRequest.new(method, false, true, uri)

data/lib/ruby-link-checker/net/http/result.rb

@@ -30,6 +30,10 @@ module LinkChecker
           request
         end
 
+        def response_headers
+          response || {}
+        end
+
         def redirect_to
           return nil unless response
 

data/lib/ruby-link-checker/result.rb

@@ -1,6 +1,10 @@
+require 'timeout'
+require 'socket'
+
 module LinkChecker
   class Result
     attr_accessor :uri, :result_uri, :method, :options, :checker
+    attr_writer :cached
 
     def initialize(current_uri, method, original_uri, options = {})
       @uri = original_uri
@@ -13,6 +17,10 @@ module LinkChecker
       false
     end
 
+    def cached?
+      !!@cached
+    end
+
     def failure?
       false
     end
@@ -33,6 +41,14 @@ module LinkChecker
       {}
     end
 
+    def response_headers
+      {}
+    end
+
+    def error_type
+      nil
+    end
+
     def code
       nil
     end
@@ -70,5 +86,14 @@ module LinkChecker
     def code
       error.class.name
     end
+
+    def error_type
+      return :timeout if error.is_a?(Timeout::Error)
+      return :timeout if defined?(::Async::TimeoutError) && error.is_a?(::Async::TimeoutError)
+      return error.error_type if error.is_a?(LinkChecker::Errors::ConnectionError)
+      return :connection if [IOError, SystemCallError, SocketError].any? { |klass| error.is_a?(klass) }
+
+      :other
+    end
   end
 end

data/lib/ruby-link-checker/task.rb

@@ -3,6 +3,7 @@ module LinkChecker
     include LinkChecker::Callbacks
 
     attr_reader :uri, :original_uri, :method, :logger, :options, :checker
+    attr_accessor :delay
 
     def initialize(checker, uri, method, original_uri, options = {})
       @checker = checker
@@ -11,6 +12,7 @@ module LinkChecker
       @original_uri = original_uri || @uri
       @method = method
       @options = options
+      @delay = 0
     end
 
     def run!

data/lib/ruby-link-checker/tasks.rb

@@ -1,3 +1,5 @@
+require 'time'
+
 module LinkChecker
   class Tasks
     include LinkChecker::Callbacks
@@ -7,6 +9,7 @@ module LinkChecker
     def initialize(checker, task_klass, uri, methods, options = {})
       @uri = uri
       @retries_left = checker.retries
+      @retry_count = 0
       @methods_left = methods.dup
       @methods = methods.dup
       @task_klass = task_klass
@@ -24,11 +27,13 @@ module LinkChecker
     def execute!
       if retry?
         @retries_left -= 1
+        @retry_count += 1
         retry! @result
-        _queue_task(uri, method, original_uri || uri, options)
+        _queue_task(uri, method, original_uri || uri, options, delay: _retry_delay)
       elsif methods_left.any?
         @method = methods_left.shift
         @uri = URI(@uri) unless @uri.is_a?(URI)
+        @redirects = [@uri] if @result.nil?
         _queue_task(uri, method, original_uri || uri, options)
       elsif @result && result.error?
         error! @result
@@ -57,11 +62,37 @@ module LinkChecker
     end
 
     def retry?
-      !first_time? && retries_left > 0
+      !first_time? && retries_left > 0 && retryable?
+    end
+
+    def retryable?
+      return false unless result
+      return result.error? || result.failure? if checker.retry_status_codes.nil?
+      return %i[timeout connection].include?(result.error_type) if result.error?
+
+      result.failure? && checker.retry_status_codes.include?(result.code)
     end
 
-    def _queue_task(uri, method, original_uri, options = {})
+    def _retry_delay
+      delay = checker.retry_after.zero? ? 0 : checker.retry_after * (checker.retry_backoff**(@retry_count - 1))
+      delay = [delay, _server_retry_delay].max if checker.respect_retry_after
+      delay *= 1 + (rand * checker.retry_jitter) if checker.retry_jitter.positive?
+      [delay, checker.max_retry_delay].min
+    end
+
+    def _server_retry_delay
+      headers = result.response_headers
+      value = Array(headers['Retry-After'] || headers['retry-after']).first.to_s.strip
+      return value.to_i if /\A\d+\z/.match?(value)
+
+      [Time.httpdate(value) - Time.now, 0].max
+    rescue ArgumentError
+      0
+    end
+
+    def _queue_task(uri, method, original_uri, options = {}, delay: 0)
       task = new_task(uri, method, original_uri, options)
+      task.delay = delay
       task.on :result do |result|
         _handle_result result
       end
@@ -88,14 +119,17 @@ module LinkChecker
       result! result
       if result.redirect?
         redirect! result
-        redirected_to_uri = URI.join(uri, result.redirect_to)
+        redirected_to_uri = URI.join(result.result_uri, result.redirect_to)
         if redirects.include?(redirected_to_uri)
           raise LinkChecker::Errors::RedirectLoopError,
                 redirects.push(redirected_to_uri)
         end
+        if redirects.length - 1 >= checker.max_redirects
+          raise LinkChecker::Errors::RedirectLimitError.new(checker.max_redirects, redirects + [redirected_to_uri])
+        end
 
         redirects << redirected_to_uri
-        _queue_task(redirected_to_uri, result.method, uri, options)
+        _queue_task(redirected_to_uri, result.method, result.uri, options)
       elsif result.success?
         success! result
       else
@@ -104,7 +138,7 @@ module LinkChecker
       end
     rescue StandardError => e
       logger.error("#{self}##{__method__}") { e } rescue nil
-      error_result = ResultError.new(result.uri, result.method, result.result_uri, e, options)
+      error_result = ResultError.new(result.result_uri, result.method, result.uri, e, options)
       result! error_result
       error! error_result
     end

data/lib/ruby-link-checker/typhoeus/hydra/checker.rb

@@ -5,7 +5,7 @@ module LinkChecker
         def run!
           request = ::Typhoeus::Request.new(
             uri, {
-              method: method,
+              method: method.downcase.to_sym,
               followlocation: false,
               timeout: checker.timeout,
               connecttimeout: checker.connecttimeout,
@@ -17,11 +17,14 @@ module LinkChecker
           request.on_complete do |response|
             if response.timed_out?
               result! ResultError.new(uri, method, original_uri, Timeout::Error.new, options)
+            elsif response.code.zero?
+              error = LinkChecker::Errors::ConnectionError.new(response.return_code, response.return_message)
+              result! ResultError.new(uri, method, original_uri, error, options)
             else
               result! Result.new(uri, method, original_uri, request, response, options)
             end
           end
-          checker._queue(request)
+          checker._queue(request, delay: delay)
         end
       end
 
@@ -35,15 +38,34 @@ module LinkChecker
             send("#{key}=", options[key] || LinkChecker::Typhoeus::Hydra::Config.send(key))
           end
           @hydra = ::Typhoeus::Hydra.new(options[:hydra] || { max_concurrency: 10 })
+          @delayed = []
           super options
         end
 
         def run
-          @hydra.run
+          loop do
+            @hydra.run
+            break if @delayed.empty?
+
+            deadline = @delayed.map(&:first).min
+            sleep([deadline - monotonic_time, 0].max)
+            ready, @delayed = @delayed.partition { |time, _request| time <= monotonic_time }
+            ready.each { |entry| @hydra.queue(entry.last) }
+          end
+        end
+
+        def _queue(request, delay: 0)
+          if delay.positive?
+            @delayed << [monotonic_time + delay, request]
+          else
+            @hydra.queue(request)
+          end
         end
 
-        def _queue(request)
-          @hydra.queue(request)
+        private
+
+        def monotonic_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
         end
       end
     end

data/lib/ruby-link-checker/typhoeus/hydra/result.rb

@@ -30,6 +30,10 @@ module LinkChecker
           request.options[:headers]
         end
 
+        def response_headers
+          response&.headers || {}
+        end
+
         def redirect_to
           return nil unless response
 

data/lib/ruby-link-checker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LinkChecker
-  VERSION = '0.4.0'
+  VERSION = '0.5.1'
 end

data/lib/ruby-link-checker.rb

@@ -9,6 +9,7 @@ require_relative 'ruby-link-checker/task'
 require_relative 'ruby-link-checker/tasks'
 require_relative 'ruby-link-checker/checker'
 require_relative 'ruby-link-checker/result'
+require_relative 'ruby-link-checker/cache'
 require_relative 'ruby-link-checker/net/http'
 require_relative 'ruby-link-checker/typhoeus/hydra'
 require_relative 'ruby-link-checker/async/http'

data/ruby-link-checker.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.platform = Gem::Platform::RUBY
   s.required_rubygems_version = '>= 1.3.6'
   s.required_ruby_version = '>= 3.3'
-  s.files = Dir['**/*']
+  s.files = IO.popen(%w[git ls-files -z], &:read).split("\0")
   s.require_paths = ['lib']
   s.homepage = 'http://github.com/dblock/ruby-link-checker'
   s.licenses = ['MIT']

data/spec/ruby-link-checker/async/http/checker_spec.rb

@@ -21,9 +21,10 @@ describe LinkChecker::Async::HTTP::Checker do
       config.hook_into :webmock
       # async-http's WebMock adapter replays responses through a real HTTP/1
       # parser, which rejects a Transfer-Encoding header alongside a
-      # Content-Length or an empty body, unlike the other checkers' adapters.
+      # Content-Length or an empty body. Let the adapter set Content-Length.
       config.before_playback do |interaction|
         interaction.response.headers.delete('Transfer-Encoding')
+        interaction.response.headers.delete_if { |key, _value| key.downcase == 'content-length' }
       end
     end
   end
@@ -32,6 +33,43 @@ describe LinkChecker::Async::HTTP::Checker do
     LinkChecker::Async::HTTP::Config.reset
   end
 
+  describe 'concurrency' do
+    let(:checker) { described_class.new(max_concurrency: 2) }
+    let(:counts) { [] }
+    let(:completed) { [] }
+
+    before do
+      active = 0
+      5.times do
+        checker._queue do
+          counts << (active += 1)
+          sleep(0.005)
+          active -= 1
+          completed << active
+        end
+      end
+    end
+
+    it 'bounds active work and waits for queued tasks' do
+      checker.run
+      expect(counts.max).to eq(2)
+      expect(counts.size).to eq(5)
+      expect(completed.size).to eq(5)
+      expect(completed.last).to eq(0)
+    end
+
+    it 'rejects invalid limits' do
+      expect { described_class.new(max_concurrency: 0) }.to raise_error ArgumentError
+    end
+
+    it 'reuses clients for the same endpoint' do
+      expect(checker._client_for(URI('https://www.example.org/one'))).to equal(
+        checker._client_for(URI('https://www.example.org/two'))
+      )
+      checker.run
+    end
+  end
+
   describe AsyncTestLinkChecker::LinkChecker do
     it_behaves_like 'a link checker'
 
@@ -60,6 +98,53 @@ describe LinkChecker::Async::HTTP::Checker do
       end
     end
 
+    context 'with a protocol option', vcr: { cassette_name: '200' } do
+      before do
+        LinkChecker::Async::HTTP.configure do |config|
+          config.protocol = protocol
+        end
+        expect(Async::HTTP::Endpoint).to receive(:parse).with(
+          URI(url).to_s,
+          hash_including(protocol: expected_protocol)
+        ).and_call_original
+      end
+
+      include_context 'with url'
+
+      context 'http1' do
+        let(:protocol) { :http1 }
+        let(:expected_protocol) { Async::HTTP::Protocol::HTTP1 }
+
+        it 'creates requests using HTTP/1' do
+          expect(result.success?).to be true
+        end
+      end
+
+      context 'http2' do
+        let(:protocol) { :http2 }
+        let(:expected_protocol) { Async::HTTP::Protocol::HTTP2 }
+
+        it 'creates requests using HTTP/2' do
+          expect(result.success?).to be true
+        end
+      end
+    end
+
+    context 'with an unsupported protocol option' do
+      before do
+        LinkChecker::Async::HTTP.configure do |config|
+          config.protocol = :http3
+        end
+      end
+
+      include_context 'with url'
+
+      it 'raises an ArgumentError' do
+        expect(result.error?).to be true
+        expect(result.error.message).to eq('Unsupported protocol: :http3, expected :http1 or :http2')
+      end
+    end
+
     context 'timeout' do
       before do
         stub_request(:get, 'https://www.example.org/').to_timeout

data/spec/ruby-link-checker/checker_spec.rb

@@ -2,9 +2,96 @@
 require 'spec_helper'
 
 describe LinkChecker::Checker do
-    context 'config' do
+  describe LinkChecker::Cache do
+    let(:directory) { Dir.mktmpdir('link-checker-spec-') }
+    let(:file) { File.join(directory, 'cache.json') }
+    let(:cache) { described_class.new({ file: file, ttl: 60 }, logger: LinkChecker::Logger.default) }
+    let(:result) do
+      LinkChecker::Net::HTTP::Result.new(
+        URI('https://www.example.org/final'), 'GET', URI('https://www.example.org/'),
+        {}, Net::HTTPOK.new('1.1', '200', 'OK'), { location: 'private.html' }
+      )
+    end
+
+    after do
+      FileUtils.remove_entry(directory)
+    end
+
+    it 'expires successful entries using the configured TTL' do
+      cache.store('key', result)
+      allow(Time).to receive(:now).and_return(Time.now + 61)
+      expect(cache.fetch('key', 'https://www.example.org/')).to be_nil
+    end
+
+    it 'retains the final URL and status without persisting metadata or headers' do
+      cache.store('key', result)
+      restored = cache.fetch('key', 'https://www.example.org/')
+      expect(restored.result_uri.to_s).to eq('https://www.example.org/final')
+      expect(restored.code).to eq(200)
+      data = JSON.parse(File.read(file)).fetch('entries').fetch('key')
+      expect(data.keys).to contain_exactly('checked_at', 'method', 'code', 'result_uri')
+    end
+
+    it 'merges writes from separate cache instances' do
+      other_cache = described_class.new({ file: file, ttl: 60 }, logger: LinkChecker::Logger.default)
+      cache.store('one', result)
+      other_cache.store('two', result)
+      expect(cache.fetch('one', 'https://www.example.org/')).not_to be_nil
+      expect(cache.fetch('two', 'https://www.example.org/')).not_to be_nil
+    end
+
+    it 'recovers from corrupted cache files' do
+      File.write(file, 'invalid JSON')
+      expect(cache.fetch('key', 'https://www.example.org/')).to be_nil
+      cache.store('key', result)
+      expect(cache.fetch('key', 'https://www.example.org/').code).to eq(200)
+    end
+
+    it 'ignores malformed cache entries' do
+      File.write(file, JSON.generate('version' => 1, 'entries' => { 'key' => true }))
+      expect(cache.fetch('key', 'https://www.example.org/')).to be_nil
+    end
+
+    it 'does not store credentials in redirect URLs' do
+      result.result_uri = URI('https://user:password@www.example.org/')
+      cache.store('key', result)
+      expect(File).not_to exist(file)
+    end
+
+    it 'requires an explicit file and TTL' do
+      expect { described_class.new({}, logger: LinkChecker::Logger.default) }.to raise_error ArgumentError
+      expect { described_class.new({ file: file }, logger: LinkChecker::Logger.default) }.to raise_error ArgumentError
+    end
+
+    it 'does not turn a cache write failure into a failed link' do
+      allow(FileUtils).to receive(:mkdir_p).and_raise(Errno::EACCES)
+      expect { cache.store('key', result) }.not_to raise_error
+    end
+  end
+
+  context 'config' do
     it 'requires at least one method' do
-      expect { LinkChecker::Checker.new(methods: []) }.to raise_error ArgumentError, 'Missing methods.'
+      expect { described_class.new(methods: []) }.to raise_error ArgumentError, 'Missing methods.'
+    end
+
+    it 'rejects an invalid redirect limit' do
+      expect { described_class.new(max_redirects: -1) }.to raise_error ArgumentError, 'Invalid redirect limit: -1'
+    end
+
+    it 'validates retry delays and backoff' do
+      expect { described_class.new(retry_after: -1) }.to raise_error ArgumentError
+      expect { described_class.new(retry_backoff: 0.5) }.to raise_error ArgumentError
+      expect { described_class.new(retry_jitter: 2) }.to raise_error ArgumentError
+      expect { described_class.new(max_retry_delay: Float::INFINITY) }.to raise_error ArgumentError
+    end
+
+    it 'rejects invalid retry status codes' do
+      expect { described_class.new(retry_status_codes: ['503']) }.to raise_error ArgumentError
+    end
+
+    it 'allows per-instance disabling of global Retry-After handling' do
+      LinkChecker.config.respect_retry_after = true
+      expect(described_class.new(respect_retry_after: false).respect_retry_after).to be false
     end
   end
 end

data/spec/ruby-link-checker/config_spec.rb

@@ -19,6 +19,12 @@ describe LinkChecker::Config do
       it 'requires a positive integer' do
         expect { LinkChecker.config.retries = -1 }.to raise_error ArgumentError, 'Invalid number of retries: -1'
       end
+
+      context 'max_redirects' do
+        it 'rejects negative limits' do
+          expect { LinkChecker.config.max_redirects = -1 }.to raise_error ArgumentError, 'Invalid redirect limit: -1'
+        end
+      end
     end
   end
 

data/spec/ruby-link-checker/net/http/checker_spec.rb

@@ -15,6 +15,21 @@ describe LinkChecker::Net::HTTP::Checker do
 
   it_behaves_like 'a link checker'
 
+  context 'with an HTTP URL' do
+    let(:checker) { described_class.new(methods: ['GET']) }
+
+    around do |example|
+      VCR.turned_off { example.run }
+    end
+
+    it 'does not enable TLS' do
+      stub_request(:get, 'http://www.example.org/').to_return(status: 200)
+      checker.check('http://www.example.org/')
+      expect(checker.results[:success].first.code).to eq(200)
+      expect(a_request(:get, 'http://www.example.org/')).to have_been_made.once
+    end
+  end
+
   describe '.config' do
     it 'returns the Config module' do
       expect(LinkChecker::Net::HTTP.config).to eq LinkChecker::Net::HTTP::Config

data/spec/ruby-link-checker/result_error_spec.rb

@@ -14,4 +14,21 @@ describe LinkChecker::ResultError do
   it 'returns the error class name as code' do
     expect(result.code).to eq 'StandardError'
   end
+
+  it 'classifies native timeouts without replacing the exception' do
+    error = Net::ReadTimeout.new('timeout')
+    result.error = error
+    expect(result.error_type).to eq(:timeout)
+    expect(result.error).to equal(error)
+  end
+
+  it 'classifies native connection failures' do
+    result.error = Errno::ECONNREFUSED.new
+    expect(result.error_type).to eq(:connection)
+  end
+
+  it 'does not classify programming errors as transient' do
+    result.error = ArgumentError.new
+    expect(result.error_type).to eq(:other)
+  end
 end

data/spec/ruby-link-checker/tasks_spec.rb

@@ -3,6 +3,50 @@
 require 'spec_helper'
 
 describe LinkChecker::Tasks do
+  describe 'retry delays' do
+    let(:checker) { LinkChecker::Net::HTTP::Checker.new }
+    let(:tasks) { described_class.new(checker, LinkChecker::Net::HTTP::Task, 'https://www.example.org', ['GET']) }
+    let(:headers) { { 'Retry-After' => '60' } }
+    let(:result) { instance_double(LinkChecker::Net::HTTP::Result, response_headers: headers) }
+
+    before do
+      allow(tasks).to receive(:result).and_return(result)
+    end
+
+    it 'parses delta-seconds' do
+      expect(tasks.send(:_server_retry_delay)).to eq(60)
+    end
+
+    context 'with a past HTTP date' do
+      let(:headers) { { 'Retry-After' => (Time.now - 60).httpdate } }
+
+      it 'does not delay a retry' do
+        expect(tasks.send(:_server_retry_delay)).to eq(0)
+      end
+    end
+
+    context 'with an invalid negative delay' do
+      let(:headers) { { 'Retry-After' => '-5' } }
+
+      it 'does not delay a retry' do
+        expect(tasks.send(:_server_retry_delay)).to eq(0)
+      end
+    end
+
+    it 'caps configured delays' do
+      checker.retry_after = 120
+      checker.max_retry_delay = 10
+      expect(tasks.send(:_retry_delay)).to eq(10)
+    end
+
+    it 'adds jitter without reducing the minimum delay' do
+      checker.retry_after = 2
+      checker.retry_jitter = 0.5
+      allow(tasks).to receive(:rand).and_return(0.5)
+      expect(tasks.send(:_retry_delay)).to eq(2.5)
+    end
+  end
+
   describe '#_handle_result' do
     # Reproduces issue #12: SystemStackError when logging causes repeated errors
     # https://github.com/dblock/ruby-link-checker/issues/12