ruby-link-checker 0.3.0 → 0.4.0

data/lib/ruby-link-checker/async/http/checker.rb

@@ -0,0 +1,92 @@
+module LinkChecker
+  module Async
+    module HTTP
+      class Task < ::LinkChecker::Task
+        def run!
+          checker._queue { _run! }
+        end
+
+        private
+
+        def _run!
+          client = checker._client_for(uri)
+          headers = ::Protocol::HTTP::Headers[
+            [['user-agent', checker.user_agent]] + checker.headers.map { |key, value| [key, value] }
+          ]
+          request = ::Protocol::HTTP::Request[method, uri.request_uri, headers]
+          response = _call(client, request)
+          result! Result.new(uri, method, original_uri, request, response, options)
+        rescue StandardError => e
+          result! ResultError.new(uri, method, original_uri, e, options)
+        ensure
+          response&.close
+        end
+
+        def _call(client, request)
+          if checker.read_timeout
+            ::Async::Task.current.with_timeout(checker.read_timeout) { client.call(request) }
+          else
+            client.call(request)
+          end
+        end
+      end
+
+      class Checker < LinkChecker::Checker
+        extend ::LinkChecker::Async::HTTP::Config
+
+        attr_accessor(*LinkChecker::Async::HTTP::Config::ATTRIBUTES)
+
+        def initialize(options = {})
+          LinkChecker::Async::HTTP::Config::ATTRIBUTES.each do |key|
+            send("#{key}=", options[key] || LinkChecker::Async::HTTP::Config.send(key))
+          end
+          @queue = []
+          @clients = {}
+          super
+        end
+
+        # Runs all queued checks concurrently and blocks until they complete.
+        def run
+          Barrier do |barrier|
+            @barrier = barrier
+            _flush!
+          end
+        ensure
+          @barrier = nil
+          _close_clients!
+        end
+
+        def _queue(&block)
+          if @barrier
+            @barrier.async(&block)
+          else
+            @queue << block
+          end
+        end
+
+        # Returns a client for the given URI's host/port/scheme, reusing (and
+        # pooling connections on) one client per endpoint across all checks.
+        def _client_for(uri)
+          key = [uri.scheme, uri.host, uri.port]
+          @clients[key] ||= begin
+            endpoint = ::Async::HTTP::Endpoint.parse(uri.to_s, timeout: open_timeout)
+            ::Async::HTTP::Client.new(endpoint)
+          end
+        end
+
+        private
+
+        def _flush!
+          queue = @queue
+          @queue = []
+          queue.each { |block| @barrier.async(&block) }
+        end
+
+        def _close_clients!
+          @clients.each_value(&:close)
+          @clients = {}
+        end
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/async/http/config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Async
+    module HTTP
+      module Config
+        extend self
+
+        ATTRIBUTES = %i[
+          read_timeout
+          open_timeout
+        ].freeze
+
+        attr_accessor(*Config::ATTRIBUTES)
+
+        def reset
+          self.read_timeout = nil
+          self.open_timeout = nil
+        end
+      end
+
+      class << self
+        def configure
+          block_given? ? yield(Config) : Config
+        end
+
+        def config
+          Config
+        end
+      end
+    end
+  end
+end
+
+LinkChecker::Async::HTTP::Config.reset

data/lib/ruby-link-checker/async/http/result.rb

@@ -0,0 +1,81 @@
+module LinkChecker
+  module Async
+    module HTTP
+      # A thin, case-insensitive wrapper around Protocol::HTTP::Headers,
+      # since Protocol::HTTP::Headers#[] expects a lower-cased key.
+      class Headers
+        def initialize(headers)
+          @headers = headers
+        end
+
+        def [](key)
+          value = @headers[key.to_s.downcase]
+          value.is_a?(Array) ? value.join(', ') : value
+        end
+      end
+
+      # Wraps an Async::HTTP response to expose a `code` and case-insensitive
+      # `headers`, similar to the other checkers' response objects.
+      class Response
+        attr_reader :status, :headers
+
+        def initialize(response)
+          @status = response.status
+          @headers = Headers.new(response.headers)
+        end
+
+        def code
+          @status.to_s
+        end
+      end
+
+      class Result < ::LinkChecker::Result
+        attr_accessor :request, :response
+
+        def initialize(uri, method, original_uri, request, response, options)
+          @request = request
+          @response = response && Response.new(response)
+          super(uri, method, original_uri, options)
+        end
+
+        def error?
+          false
+        end
+
+        def failure?
+          !success? && !redirect?
+        end
+
+        def code
+          @code ||= begin
+            response.status
+          rescue StandardError
+            -1
+          end
+        end
+
+        def request_headers
+          Headers.new(request.headers)
+        end
+
+        def redirect_to
+          return nil unless response
+
+          response.headers['location']
+        end
+
+        def redirect?
+          return false unless response
+
+          [301, 302, 303, 307, 308].include?(code)
+        end
+
+        def success?
+          return false unless response
+
+          code >= 200 && code <= 299
+        end
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/async/http.rb

@@ -0,0 +1,3 @@
+require_relative 'http/config'
+require_relative 'http/result'
+require_relative 'http/checker'

data/lib/ruby-link-checker/callbacks.rb

@@ -20,9 +20,9 @@ module LinkChecker
       end
     end
 
-    def method_missing(m, *args, &block)
+    def method_missing(m, *, &)
       if m.to_s[-1] == '!'
-        callback(m.to_s[...-1].to_sym, *args)
+        callback(m.to_s[...-1].to_sym, *)
       else
         super
       end

data/lib/ruby-link-checker/config.rb

@@ -7,6 +7,7 @@ module LinkChecker
     ATTRIBUTES = %i[
       methods
       user_agent
+      headers
       logger
       retries
     ].freeze
@@ -16,6 +17,7 @@ module LinkChecker
     def reset
       self.methods = %w[HEAD GET]
       self.user_agent = "Ruby Link Checker/#{LinkChecker::VERSION}"
+      self.headers = {}
       self.logger = nil
       self.retries = 0
     end

data/lib/ruby-link-checker/net/http/checker.rb

@@ -8,6 +8,7 @@ module LinkChecker
             http.open_timeout = checker.open_timeout if checker.open_timeout
             request = ::Net::HTTPGenericRequest.new(method, false, true, uri)
             request['User-Agent'] = checker.user_agent
+            checker.headers.each { |key, value| request[key] = value }
             response = http.request(request)
             result! Result.new(uri, method, original_uri, request, response, options)
           end
@@ -16,6 +17,7 @@ module LinkChecker
 
       class Checker < LinkChecker::Checker
         extend ::LinkChecker::Net::HTTP::Config
+
         attr_accessor(*LinkChecker::Net::HTTP::Config::ATTRIBUTES)
 
         def initialize(options = {})

data/lib/ruby-link-checker/tasks.rb

@@ -103,8 +103,10 @@ module LinkChecker
         execute!
       end
     rescue StandardError => e
-      logger.error("#{self}##{__method__}") { e }
-      _handle_result ResultError.new(result.uri, result.method, result.result_uri, e, options)
+      logger.error("#{self}##{__method__}") { e } rescue nil
+      error_result = ResultError.new(result.uri, result.method, result.result_uri, e, options)
+      result! error_result
+      error! error_result
     end
   end
 end

data/lib/ruby-link-checker/typhoeus/hydra/checker.rb

@@ -11,7 +11,7 @@ module LinkChecker
               connecttimeout: checker.connecttimeout,
               headers: {
                 'User-Agent' => checker.user_agent
-              }
+              }.merge(checker.headers)
             }
           )
           request.on_complete do |response|
@@ -27,6 +27,7 @@ module LinkChecker
 
       class Checker < LinkChecker::Checker
         extend ::LinkChecker::Typhoeus::Hydra::Config
+
         attr_accessor(*LinkChecker::Typhoeus::Hydra::Config::ATTRIBUTES)
 
         def initialize(options = {})

data/lib/ruby-link-checker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LinkChecker
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/ruby-link-checker.rb

@@ -11,3 +11,4 @@ require_relative 'ruby-link-checker/checker'
 require_relative 'ruby-link-checker/result'
 require_relative 'ruby-link-checker/net/http'
 require_relative 'ruby-link-checker/typhoeus/hydra'
+require_relative 'ruby-link-checker/async/http'

data/ruby-link-checker.gemspec

@@ -10,12 +10,13 @@ Gem::Specification.new do |s|
   s.email = 'dblock@dblock.org'
   s.platform = Gem::Platform::RUBY
   s.required_rubygems_version = '>= 1.3.6'
-  s.required_ruby_version = '>= 2.7'
+  s.required_ruby_version = '>= 3.3'
   s.files = Dir['**/*']
   s.require_paths = ['lib']
   s.homepage = 'http://github.com/dblock/ruby-link-checker'
   s.licenses = ['MIT']
   s.summary = 'Fast ruby link checker.'
   s.metadata['rubygems_mfa_required'] = 'true'
+  s.add_development_dependency 'async-http'
   s.add_development_dependency 'typhoeus'
 end

data/spec/ruby-link-checker/async/http/checker_spec.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'async'
+require 'async/http'
+
+describe LinkChecker::Async::HTTP::Checker do
+  module AsyncTestLinkChecker
+    class Task < LinkChecker::Async::HTTP::Task; end
+
+    class LinkChecker < LinkChecker::Async::HTTP::Checker
+      def check(url, options = {})
+        super
+        run
+      end
+    end
+  end
+
+  before :all do
+    VCR.configure do |config|
+      config.hook_into :webmock
+      # async-http's WebMock adapter replays responses through a real HTTP/1
+      # parser, which rejects a Transfer-Encoding header alongside a
+      # Content-Length or an empty body, unlike the other checkers' adapters.
+      config.before_playback do |interaction|
+        interaction.response.headers.delete('Transfer-Encoding')
+      end
+    end
+  end
+
+  after do
+    LinkChecker::Async::HTTP::Config.reset
+  end
+
+  describe AsyncTestLinkChecker::LinkChecker do
+    it_behaves_like 'a link checker'
+
+    describe '.config' do
+      it 'returns the Config module' do
+        expect(LinkChecker::Async::HTTP.config).to eq LinkChecker::Async::HTTP::Config
+      end
+    end
+
+    context 'with timeout options', vcr: { cassette_name: '200' } do
+      before do
+        LinkChecker::Async::HTTP.configure do |config|
+          config.read_timeout = 5
+          config.open_timeout = 10
+        end
+        expect(Async::HTTP::Endpoint).to receive(:parse).with(
+          URI(url).to_s,
+          hash_including(timeout: 10)
+        ).and_call_original
+      end
+
+      include_context 'with url'
+
+      it 'creates requests with a default timeout' do
+        expect(result.success?).to be true
+      end
+    end
+
+    context 'timeout' do
+      before do
+        stub_request(:get, 'https://www.example.org/').to_timeout
+      end
+
+      include_context 'with url'
+
+      around do |example|
+        VCR.turned_off { example.run }
+      end
+
+      it 'times out' do
+        expect(result.success?).to be false
+        expect(result.error?).to be true
+      end
+
+      context 'with metadata' do
+        let(:options) { { foo: :bar } }
+
+        it 'times out' do
+          expect(result.error?).to be true
+          expect(result.options).to eq(foo: :bar)
+        end
+      end
+    end
+  end
+
+  describe LinkChecker::Async::HTTP::Result do
+    describe '#code' do
+      it 'returns -1 when there is no response' do
+        request = instance_double(Protocol::HTTP::Request)
+        result = described_class.new(URI('https://www.example.org'), 'GET', nil, request, nil, {})
+        expect(result.code).to eq(-1)
+      end
+    end
+  end
+end

data/spec/ruby-link-checker/callbacks_spec.rb

@@ -6,6 +6,10 @@ describe LinkChecker::Callbacks do
   subject do
     Class.new do
       include LinkChecker::Callbacks
+
+      def logger
+        @logger ||= Logger.new(nil)
+      end
     end.new
   end
 
@@ -40,4 +44,33 @@ describe LinkChecker::Callbacks do
       expect(subject).to have_received(:called!).with(:bar)
     end
   end
+
+  context 'method_missing' do
+    it 'invokes callback for a bang method' do
+      allow(subject).to receive(:called!)
+      subject.on :foo do |data|
+        subject.called! data
+      end
+      subject.foo! :foo
+      expect(subject).to have_received(:called!).with(:foo)
+    end
+
+    it 'raises NoMethodError for a non-bang method' do
+      expect { subject.foo }.to raise_error NoMethodError
+    end
+  end
+
+  context 'when a callback raises an error' do
+    before do
+      subject.on :foo do
+        raise StandardError, 'oops'
+      end
+    end
+
+    it 'rescues and logs the error' do
+      allow(subject.logger).to receive(:error)
+      expect(subject.send(:callback, :foo)).to be false
+      expect(subject.logger).to have_received(:error)
+    end
+  end
 end

data/spec/ruby-link-checker/errors/redirect_loop_error_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe LinkChecker::Errors::RedirectLoopError do
+  subject(:error) do
+    described_class.new(['https://www.example.org', 'https://www.example.org/redirect'])
+  end
+
+  it 'returns the last url' do
+    expect(error.url).to eq 'https://www.example.org/redirect'
+  end
+
+  it 'formats a message with all urls' do
+    expect(error.message)
+      .to eq 'Redirect loop: https://www.example.org -> https://www.example.org/redirect.'
+  end
+end

data/spec/ruby-link-checker/net/http/checker_spec.rb

@@ -15,6 +15,12 @@ describe LinkChecker::Net::HTTP::Checker do
 
   it_behaves_like 'a link checker'
 
+  describe '.config' do
+    it 'returns the Config module' do
+      expect(LinkChecker::Net::HTTP.config).to eq LinkChecker::Net::HTTP::Config
+    end
+  end
+
   context 'with timeout options', vcr: { cassette_name: '200' } do
     before do
       LinkChecker::Net::HTTP.configure do |config|
@@ -58,4 +64,15 @@ describe LinkChecker::Net::HTTP::Checker do
       end
     end
   end
+
+  describe LinkChecker::Net::HTTP::Result do
+    describe '#code' do
+      it 'returns -1 when the response raises' do
+        response = instance_double(Net::HTTPResponse)
+        allow(response).to receive(:code).and_raise(StandardError)
+        result = described_class.new(URI('https://www.example.org'), 'GET', nil, nil, response, {})
+        expect(result.code).to eq(-1)
+      end
+    end
+  end
 end

data/spec/ruby-link-checker/result_error_spec.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe LinkChecker::ResultError do
+  subject(:result) do
+    described_class.new('https://www.example.org', 'GET', 'https://www.example.org', StandardError.new('oops'))
+  end
+
+  it 'is an error' do
+    expect(result.error?).to be true
+  end
+
+  it 'returns the error class name as code' do
+    expect(result.code).to eq 'StandardError'
+  end
+end

data/spec/ruby-link-checker/result_spec.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe LinkChecker::Result do
+  subject(:result) do
+    described_class.new('https://www.example.org', 'GET', 'https://www.example.org')
+  end
+
+  it 'defaults to not successful' do
+    expect(result.success?).to be false
+  end
+
+  it 'defaults to not a failure' do
+    expect(result.failure?).to be false
+  end
+
+  it 'defaults to not an error' do
+    expect(result.error?).to be false
+  end
+
+  it 'defaults to not a redirect' do
+    expect(result.redirect?).to be false
+  end
+
+  it 'defaults redirect_to to nil' do
+    expect(result.redirect_to).to be_nil
+  end
+
+  it 'defaults request_headers to an empty hash' do
+    expect(result.request_headers).to eq({})
+  end
+
+  it 'defaults code to nil' do
+    expect(result.code).to be_nil
+  end
+
+  it 'defaults error to nil' do
+    expect(result.error).to be_nil
+  end
+end

data/spec/ruby-link-checker/task_spec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe LinkChecker::Task do
+  subject(:task) do
+    described_class.new(
+      LinkChecker::Net::HTTP::Checker.new,
+      'https://www.example.org',
+      'GET',
+      'https://www.example.org'
+    )
+  end
+
+  describe '#run!' do
+    it 'raises NotImplementedError' do
+      expect { task.run! }.to raise_error NotImplementedError
+    end
+  end
+end