ruby-link-checker 0.3.0 → 0.5.1

data/lib/ruby-link-checker/cache.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'tempfile'
+require 'uri'
+
+module LinkChecker
+  class CachedResult < Result
+    attr_reader :code
+
+    def initialize(data, uri)
+      @code = data.fetch('code')
+      super(URI(data.fetch('result_uri')), data.fetch('method'), URI(uri.to_s))
+      @cached = true
+    end
+
+    def success?
+      true
+    end
+  end
+
+  class Cache
+    VERSION = 1
+
+    def initialize(options, logger:)
+      unless options.is_a?(Hash) && options[:file].is_a?(String) && !options[:file].strip.empty?
+        raise ArgumentError, 'Cache requires a file path'
+      end
+
+      @ttl = options[:ttl]
+      unless @ttl.is_a?(Numeric) && @ttl.real? && @ttl.finite? && @ttl.positive?
+        raise ArgumentError, 'Cache requires a positive, finite TTL in seconds'
+      end
+
+      @file = File.expand_path(options[:file])
+      @logger = logger
+    end
+
+    def fetch(key, uri)
+      data = read_entries[key]
+      return unless fresh?(data) && data['code'].is_a?(Integer) && data['code'].between?(200, 299)
+      return unless data['method'].is_a?(String) && data['result_uri'].is_a?(String)
+
+      parsed = URI(data['result_uri'])
+      return unless %w[http https].include?(parsed.scheme) && parsed.host && !parsed.userinfo
+
+      CachedResult.new(data, uri)
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def store(key, result)
+      return unless result.success?
+      return if URI(result.result_uri.to_s).userinfo
+
+      directory = File.dirname(@file)
+      FileUtils.mkdir_p(directory)
+      File.open("#{@file}.lock", File::RDWR | File::CREAT, 0o600) do |lock|
+        lock.flock(File::LOCK_EX)
+        entries = read_entries.select { |_entry_key, data| fresh?(data) }
+        entries[key] = {
+          'checked_at' => Time.now.to_f,
+          'method' => result.method,
+          'code' => result.code,
+          'result_uri' => result.result_uri.to_s
+        }
+        Tempfile.create(['link-checker-', '.json'], directory) do |file|
+          file.write(JSON.generate('version' => VERSION, 'entries' => entries))
+          file.flush
+          File.rename(file.path, @file)
+        end
+      end
+    rescue IOError, SystemCallError, JSON::GeneratorError
+      @logger.warn("Unable to write link cache #{@file}; result was not cached")
+    end
+
+    private
+
+    def fresh?(data)
+      return false unless data.is_a?(Hash)
+
+      timestamp = data['checked_at']
+      now = Time.now.to_f
+      timestamp.is_a?(Numeric) && timestamp.finite? && timestamp <= now && now - timestamp < @ttl
+    end
+
+    def read_entries
+      data = JSON.parse(File.read(@file))
+      return {} unless data.is_a?(Hash) && data['version'] == VERSION && data['entries'].is_a?(Hash)
+
+      data['entries']
+    rescue Errno::ENOENT
+      {}
+    rescue IOError, SystemCallError, JSON::ParserError
+      @logger.warn("Unable to read link cache #{@file}; checking links normally")
+      {}
+    end
+  end
+end

data/lib/ruby-link-checker/callbacks.rb

@@ -20,9 +20,9 @@ module LinkChecker
       end
     end
 
-    def method_missing(m, *args, &block)
+    def method_missing(m, *, &)
       if m.to_s[-1] == '!'
-        callback(m.to_s[...-1].to_sym, *args)
+        callback(m.to_s[...-1].to_sym, *)
       else
         super
       end

data/lib/ruby-link-checker/checker.rb

@@ -6,15 +6,17 @@ module LinkChecker
     include LinkChecker::Callbacks
 
     attr_reader :results
-    attr_accessor(*Config::ATTRIBUTES)
 
     def initialize(options = {})
       LinkChecker::Config::ATTRIBUTES.each do |key|
-        send("#{key}=", options[key] || LinkChecker.config.send(key))
+        send("#{key}=", options.fetch(key) { LinkChecker.config.send(key) })
       end
       raise ArgumentError, "Missing methods." if methods&.none?
       @logger ||= options[:logger] || LinkChecker::Config.logger || LinkChecker::Logger.default
       @results = { error: [], failure: [], success: [] } unless options.key?(:results) && !options[:results]
+      @pending = {}
+      @successful = {}
+      @cache_store = Cache.new(cache, logger: @logger) if cache
     end
 
     def task_klass
@@ -25,6 +27,37 @@ module LinkChecker
     end
 
     def check(uri, options = {})
+      return execute_check(uri, options) unless deduplicate || @cache_store
+
+      key = request_key(uri)
+      return execute_check(uri, options) unless key
+
+      subscriber = [uri.to_s.dup, options]
+      if deduplicate && (entry = @pending[key])
+        entry[:subscribers] << subscriber
+        entry[:events].each { |event, result| publish(event, copy_result(result, subscriber)) }
+        return
+      end
+
+      cached_result = @successful[key] if deduplicate
+      cached_result ||= @cache_store&.fetch(key, uri)
+      if cached_result
+        @successful[key] ||= cached_result if deduplicate
+        result = copy_result(cached_result, subscriber)
+        result.cached = true
+        publish(:result, result)
+        publish(:success, result)
+        return
+      end
+
+      entry = { subscribers: [subscriber], events: [] }
+      @pending[key] = entry if deduplicate
+      execute_check(uri, options) { |event, result| handle_shared_result(key, entry, uri, event, result) }
+    end
+
+    private
+
+    def execute_check(uri, options, &handler)
       tasks = Tasks.new(
         self,
         task_klass,
@@ -33,10 +66,49 @@ module LinkChecker
         options
       )
       tasks.on do |event, *args|
-        results[event] << args.first if @results && %i[error failure success].include?(event)
-        callback event, *args
+        handler ? handler.call(event, *args) : publish(event, args.first)
       end
       tasks.execute!
     end
+
+    def publish(event, result)
+      result.checker = self
+      results[event] << result if @results && %i[error failure success].include?(event)
+      callback(event, result)
+    end
+
+    def handle_shared_result(key, entry, uri, event, result)
+      entry[:events] << [event, result]
+      subscribers = entry[:subscribers].dup
+      if %i[error failure success].include?(event)
+        @pending.delete(key) if @pending[key].equal?(entry)
+        if event == :success && request_key(uri) == key
+          @successful[key] = result if deduplicate
+          @cache_store&.store(key, result)
+        end
+      end
+      subscribers.each { |subscriber| publish(event, copy_result(result, subscriber)) }
+    end
+
+    def copy_result(result, subscriber)
+      result.dup.tap do |copy|
+        copy.uri = URI(subscriber.first)
+        copy.result_uri = result.result_uri.dup
+        copy.options = subscriber.last
+      end
+    end
+
+    def request_key(uri)
+      parsed = URI(uri.to_s)
+      return unless %w[http https].include?(parsed.scheme) && parsed.host && !parsed.userinfo
+
+      attributes = Config::ATTRIBUTES - %i[logger cache deduplicate]
+      attributes += %i[timeout connecttimeout read_timeout open_timeout protocol].select { |key| respond_to?(key) }
+      settings = attributes.to_h { |key| [key, send(key)] }
+      settings[:headers] = headers.sort_by { |key, _value| key.to_s }
+      Digest::SHA256.hexdigest(JSON.generate([task_klass.name, parsed.to_s, settings]))
+    rescue URI::InvalidURIError, JSON::GeneratorError, TypeError
+      nil
+    end
   end
 end

data/lib/ruby-link-checker/config.rb

@@ -7,8 +7,18 @@ module LinkChecker
     ATTRIBUTES = %i[
       methods
       user_agent
+      headers
       logger
       retries
+      max_redirects
+      retry_status_codes
+      retry_after
+      retry_backoff
+      retry_jitter
+      max_retry_delay
+      respect_retry_after
+      deduplicate
+      cache
     ].freeze
 
     attr_accessor(*Config::ATTRIBUTES)
@@ -16,8 +26,18 @@ module LinkChecker
     def reset
       self.methods = %w[HEAD GET]
       self.user_agent = "Ruby Link Checker/#{LinkChecker::VERSION}"
+      self.headers = {}
       self.logger = nil
       self.retries = 0
+      self.max_redirects = 20
+      self.retry_status_codes = [408, 425, 429, 500, 502, 503, 504]
+      self.retry_after = 0
+      self.retry_backoff = 1
+      self.retry_jitter = 0
+      self.max_retry_delay = 60
+      self.respect_retry_after = false
+      self.deduplicate = false
+      self.cache = nil
     end
 
     def retries=(value)
@@ -25,6 +45,50 @@ module LinkChecker
 
       @retries = value
     end
+
+    def max_redirects=(value)
+      raise ArgumentError, "Invalid redirect limit: #{value}" unless value.is_a?(Integer) && value >= 0
+
+      @max_redirects = value
+    end
+
+    def retry_status_codes=(value)
+      unless value.nil? || (value.is_a?(Array) && value.all? { |code| code.is_a?(Integer) && code.between?(100, 599) })
+        raise ArgumentError, "Invalid retry status codes: #{value}"
+      end
+
+      @retry_status_codes = value&.dup
+    end
+
+    def retry_after=(value)
+      validate_number!(:retry_after, value, 0)
+      @retry_after = value
+    end
+
+    def retry_backoff=(value)
+      validate_number!(:retry_backoff, value, 1)
+      @retry_backoff = value
+    end
+
+    def retry_jitter=(value)
+      validate_number!(:retry_jitter, value, 0)
+      raise ArgumentError, "Invalid retry_jitter: #{value}" if value > 1
+
+      @retry_jitter = value
+    end
+
+    def max_retry_delay=(value)
+      validate_number!(:max_retry_delay, value, 0)
+      @max_retry_delay = value
+    end
+
+    private
+
+    def validate_number!(name, value, minimum)
+      return if value.is_a?(Numeric) && value.real? && value.finite? && value >= minimum
+
+      raise ArgumentError, "Invalid #{name}: #{value}"
+    end
   end
 
   class << self

data/lib/ruby-link-checker/errors/connection_error.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Errors
+    class ConnectionError < BaseError
+      TRANSIENT_CODES = %i[
+        couldnt_resolve_proxy couldnt_resolve_host couldnt_connect
+        partial_file got_nothing send_error recv_error http2 http2_stream
+      ].freeze
+
+      attr_reader :return_code
+
+      def initialize(return_code, message)
+        @return_code = return_code
+        super(message)
+      end
+
+      def error_type
+        TRANSIENT_CODES.include?(return_code) ? :connection : :other
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/errors/redirect_limit_error.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Errors
+    class RedirectLimitError < BaseError
+      attr_reader :limit, :urls
+
+      def initialize(limit, urls)
+        @limit = limit
+        @urls = urls
+        super("Redirect limit of #{limit} exceeded: #{urls.join(' -> ')}.")
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/errors.rb

@@ -1,2 +1,4 @@
 require_relative 'errors/base_error'
 require_relative 'errors/redirect_loop_error'
+require_relative 'errors/redirect_limit_error'
+require_relative 'errors/connection_error'

data/lib/ruby-link-checker/net/http/checker.rb

@@ -3,11 +3,13 @@ module LinkChecker
     module HTTP
       class Task < ::LinkChecker::Task
         def run!
-          ::Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+          sleep(delay) if delay.positive?
+          ::Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
             http.read_timeout = checker.read_timeout if checker.read_timeout
             http.open_timeout = checker.open_timeout if checker.open_timeout
             request = ::Net::HTTPGenericRequest.new(method, false, true, uri)
             request['User-Agent'] = checker.user_agent
+            checker.headers.each { |key, value| request[key] = value }
             response = http.request(request)
             result! Result.new(uri, method, original_uri, request, response, options)
           end
@@ -16,6 +18,7 @@ module LinkChecker
 
       class Checker < LinkChecker::Checker
         extend ::LinkChecker::Net::HTTP::Config
+
         attr_accessor(*LinkChecker::Net::HTTP::Config::ATTRIBUTES)
 
         def initialize(options = {})

data/lib/ruby-link-checker/net/http/result.rb

@@ -30,6 +30,10 @@ module LinkChecker
           request
         end
 
+        def response_headers
+          response || {}
+        end
+
         def redirect_to
           return nil unless response
 

data/lib/ruby-link-checker/result.rb

@@ -1,6 +1,10 @@
+require 'timeout'
+require 'socket'
+
 module LinkChecker
   class Result
     attr_accessor :uri, :result_uri, :method, :options, :checker
+    attr_writer :cached
 
     def initialize(current_uri, method, original_uri, options = {})
       @uri = original_uri
@@ -13,6 +17,10 @@ module LinkChecker
       false
     end
 
+    def cached?
+      !!@cached
+    end
+
     def failure?
       false
     end
@@ -33,6 +41,14 @@ module LinkChecker
       {}
     end
 
+    def response_headers
+      {}
+    end
+
+    def error_type
+      nil
+    end
+
     def code
       nil
     end
@@ -70,5 +86,14 @@ module LinkChecker
     def code
       error.class.name
     end
+
+    def error_type
+      return :timeout if error.is_a?(Timeout::Error)
+      return :timeout if defined?(::Async::TimeoutError) && error.is_a?(::Async::TimeoutError)
+      return error.error_type if error.is_a?(LinkChecker::Errors::ConnectionError)
+      return :connection if [IOError, SystemCallError, SocketError].any? { |klass| error.is_a?(klass) }
+
+      :other
+    end
   end
 end

data/lib/ruby-link-checker/task.rb

@@ -3,6 +3,7 @@ module LinkChecker
     include LinkChecker::Callbacks
 
     attr_reader :uri, :original_uri, :method, :logger, :options, :checker
+    attr_accessor :delay
 
     def initialize(checker, uri, method, original_uri, options = {})
       @checker = checker
@@ -11,6 +12,7 @@ module LinkChecker
       @original_uri = original_uri || @uri
       @method = method
       @options = options
+      @delay = 0
     end
 
     def run!

data/lib/ruby-link-checker/tasks.rb

@@ -1,3 +1,5 @@
+require 'time'
+
 module LinkChecker
   class Tasks
     include LinkChecker::Callbacks
@@ -7,6 +9,7 @@ module LinkChecker
     def initialize(checker, task_klass, uri, methods, options = {})
       @uri = uri
       @retries_left = checker.retries
+      @retry_count = 0
       @methods_left = methods.dup
       @methods = methods.dup
       @task_klass = task_klass
@@ -24,11 +27,13 @@ module LinkChecker
     def execute!
       if retry?
         @retries_left -= 1
+        @retry_count += 1
         retry! @result
-        _queue_task(uri, method, original_uri || uri, options)
+        _queue_task(uri, method, original_uri || uri, options, delay: _retry_delay)
       elsif methods_left.any?
         @method = methods_left.shift
         @uri = URI(@uri) unless @uri.is_a?(URI)
+        @redirects = [@uri] if @result.nil?
         _queue_task(uri, method, original_uri || uri, options)
       elsif @result && result.error?
         error! @result
@@ -57,11 +62,37 @@ module LinkChecker
     end
 
     def retry?
-      !first_time? && retries_left > 0
+      !first_time? && retries_left > 0 && retryable?
+    end
+
+    def retryable?
+      return false unless result
+      return result.error? || result.failure? if checker.retry_status_codes.nil?
+      return %i[timeout connection].include?(result.error_type) if result.error?
+
+      result.failure? && checker.retry_status_codes.include?(result.code)
+    end
+
+    def _retry_delay
+      delay = checker.retry_after.zero? ? 0 : checker.retry_after * (checker.retry_backoff**(@retry_count - 1))
+      delay = [delay, _server_retry_delay].max if checker.respect_retry_after
+      delay *= 1 + (rand * checker.retry_jitter) if checker.retry_jitter.positive?
+      [delay, checker.max_retry_delay].min
+    end
+
+    def _server_retry_delay
+      headers = result.response_headers
+      value = Array(headers['Retry-After'] || headers['retry-after']).first.to_s.strip
+      return value.to_i if /\A\d+\z/.match?(value)
+
+      [Time.httpdate(value) - Time.now, 0].max
+    rescue ArgumentError
+      0
     end
 
-    def _queue_task(uri, method, original_uri, options = {})
+    def _queue_task(uri, method, original_uri, options = {}, delay: 0)
       task = new_task(uri, method, original_uri, options)
+      task.delay = delay
       task.on :result do |result|
         _handle_result result
       end
@@ -88,14 +119,17 @@ module LinkChecker
       result! result
       if result.redirect?
         redirect! result
-        redirected_to_uri = URI.join(uri, result.redirect_to)
+        redirected_to_uri = URI.join(result.result_uri, result.redirect_to)
         if redirects.include?(redirected_to_uri)
           raise LinkChecker::Errors::RedirectLoopError,
                 redirects.push(redirected_to_uri)
         end
+        if redirects.length - 1 >= checker.max_redirects
+          raise LinkChecker::Errors::RedirectLimitError.new(checker.max_redirects, redirects + [redirected_to_uri])
+        end
 
         redirects << redirected_to_uri
-        _queue_task(redirected_to_uri, result.method, uri, options)
+        _queue_task(redirected_to_uri, result.method, result.uri, options)
       elsif result.success?
         success! result
       else
@@ -103,8 +137,10 @@ module LinkChecker
         execute!
       end
     rescue StandardError => e
-      logger.error("#{self}##{__method__}") { e }
-      _handle_result ResultError.new(result.uri, result.method, result.result_uri, e, options)
+      logger.error("#{self}##{__method__}") { e } rescue nil
+      error_result = ResultError.new(result.result_uri, result.method, result.uri, e, options)
+      result! error_result
+      error! error_result
     end
   end
 end

data/lib/ruby-link-checker/typhoeus/hydra/checker.rb

@@ -5,28 +5,32 @@ module LinkChecker
         def run!
           request = ::Typhoeus::Request.new(
             uri, {
-              method: method,
+              method: method.downcase.to_sym,
               followlocation: false,
               timeout: checker.timeout,
               connecttimeout: checker.connecttimeout,
               headers: {
                 'User-Agent' => checker.user_agent
-              }
+              }.merge(checker.headers)
             }
           )
           request.on_complete do |response|
             if response.timed_out?
               result! ResultError.new(uri, method, original_uri, Timeout::Error.new, options)
+            elsif response.code.zero?
+              error = LinkChecker::Errors::ConnectionError.new(response.return_code, response.return_message)
+              result! ResultError.new(uri, method, original_uri, error, options)
             else
               result! Result.new(uri, method, original_uri, request, response, options)
             end
           end
-          checker._queue(request)
+          checker._queue(request, delay: delay)
         end
       end
 
       class Checker < LinkChecker::Checker
         extend ::LinkChecker::Typhoeus::Hydra::Config
+
         attr_accessor(*LinkChecker::Typhoeus::Hydra::Config::ATTRIBUTES)
 
         def initialize(options = {})
@@ -34,15 +38,34 @@ module LinkChecker
             send("#{key}=", options[key] || LinkChecker::Typhoeus::Hydra::Config.send(key))
           end
           @hydra = ::Typhoeus::Hydra.new(options[:hydra] || { max_concurrency: 10 })
+          @delayed = []
           super options
         end
 
         def run
-          @hydra.run
+          loop do
+            @hydra.run
+            break if @delayed.empty?
+
+            deadline = @delayed.map(&:first).min
+            sleep([deadline - monotonic_time, 0].max)
+            ready, @delayed = @delayed.partition { |time, _request| time <= monotonic_time }
+            ready.each { |entry| @hydra.queue(entry.last) }
+          end
         end
 
-        def _queue(request)
-          @hydra.queue(request)
+        def _queue(request, delay: 0)
+          if delay.positive?
+            @delayed << [monotonic_time + delay, request]
+          else
+            @hydra.queue(request)
+          end
+        end
+
+        private
+
+        def monotonic_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
         end
       end
     end

data/lib/ruby-link-checker/typhoeus/hydra/result.rb

@@ -30,6 +30,10 @@ module LinkChecker
           request.options[:headers]
         end
 
+        def response_headers
+          response&.headers || {}
+        end
+
         def redirect_to
           return nil unless response
 

data/lib/ruby-link-checker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LinkChecker
-  VERSION = '0.3.0'
+  VERSION = '0.5.1'
 end

data/lib/ruby-link-checker.rb

@@ -9,5 +9,7 @@ require_relative 'ruby-link-checker/task'
 require_relative 'ruby-link-checker/tasks'
 require_relative 'ruby-link-checker/checker'
 require_relative 'ruby-link-checker/result'
+require_relative 'ruby-link-checker/cache'
 require_relative 'ruby-link-checker/net/http'
 require_relative 'ruby-link-checker/typhoeus/hydra'
+require_relative 'ruby-link-checker/async/http'

data/ruby-link-checker.gemspec

@@ -10,12 +10,13 @@ Gem::Specification.new do |s|
   s.email = 'dblock@dblock.org'
   s.platform = Gem::Platform::RUBY
   s.required_rubygems_version = '>= 1.3.6'
-  s.required_ruby_version = '>= 2.7'
-  s.files = Dir['**/*']
+  s.required_ruby_version = '>= 3.3'
+  s.files = IO.popen(%w[git ls-files -z], &:read).split("\0")
   s.require_paths = ['lib']
   s.homepage = 'http://github.com/dblock/ruby-link-checker'
   s.licenses = ['MIT']
   s.summary = 'Fast ruby link checker.'
   s.metadata['rubygems_mfa_required'] = 'true'
+  s.add_development_dependency 'async-http'
   s.add_development_dependency 'typhoeus'
 end