ruby-link-checker 0.1.0

data/lib/ruby-link-checker/callbacks.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Callbacks
+    def callbacks
+      @callbacks ||= Hash.new { |h, k| h[k] = [] }
+    end
+
+    def delegates
+      @delegates ||= []
+    end
+
+    def on(*events, &block)
+      if events && Array(events).any?
+        Array(events).each do |event|
+          callbacks[event.to_s] << block
+        end
+      else
+        delegates << block
+      end
+    end
+
+    def method_missing(m, *args, &block)
+      if m.to_s[-1] == '!'
+        callback(m.to_s[...-1].to_sym, *args)
+      else
+        super
+      end
+    end
+
+    private
+
+    def callback(event, *data)
+      delegates.each do |c|
+        c.call(event, *data)
+      end
+
+      callbacks = self.callbacks[event.to_s]
+      return false unless callbacks
+
+      callbacks.each do |c|
+        c.call(*data)
+      end
+      true
+    rescue StandardError => e
+      logger.error("#{self}##{__method__}") { e }
+      false
+    end
+  end
+end

data/lib/ruby-link-checker/checker.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  class Checker
+    include LinkChecker::Config
+    include LinkChecker::Callbacks
+
+    attr_reader :results
+    attr_accessor(*Config::ATTRIBUTES)
+
+    def initialize(options = {})
+      LinkChecker::Config::ATTRIBUTES.each do |key|
+        send("#{key}=", options[key] || LinkChecker.config.send(key))
+      end
+      raise ArgumentError, "Missing methods." if methods&.none?
+      @logger ||= options[:logger] || LinkChecker::Config.logger || LinkChecker::Logger.default
+      @results = { error: [], failure: [], success: [] } unless options.key?(:results) && !options[:results]
+    end
+
+    def task_klass
+      @task_klass ||= begin
+        module_name = self.class.name.split("::")[...-1].join('::')
+        Object.const_get("#{module_name}::Task")
+      end
+    end
+
+    def check(uri, options = {})
+      tasks = Tasks.new(
+        self,
+        task_klass,
+        uri,
+        methods,
+        options
+      )
+      tasks.on do |event, *args|
+        results[event] << args.first if @results && %i[error failure success].include?(event)
+        callback event, *args
+      end
+      tasks.execute!
+    end
+  end
+end

data/lib/ruby-link-checker/config.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Config
+    extend self
+
+    ATTRIBUTES = %i[
+      methods
+      user_agent
+      logger
+      retries
+    ].freeze
+
+    attr_accessor(*Config::ATTRIBUTES)
+
+    def reset
+      self.methods = %w[HEAD GET]
+      self.user_agent = "Ruby Link Checker/#{LinkChecker::VERSION}"
+      self.logger = nil
+      self.retries = 0
+    end
+
+    def retries=(value)
+      raise ArgumentError, "Invalid number of retries: #{value}" unless value.is_a?(Integer) && value >= 0
+
+      @retries = value
+    end
+  end
+
+  class << self
+    def configure
+      block_given? ? yield(Config) : Config
+    end
+
+    def config
+      Config
+    end
+  end
+end
+
+LinkChecker::Config.reset

data/lib/ruby-link-checker/errors/base_error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Errors
+    class BaseError < StandardError
+    end
+  end
+end

data/lib/ruby-link-checker/errors/redirect_loop_error.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Errors
+    class RedirectLoopError < BaseError
+      attr_accessor :urls
+
+      def initialize(urls)
+        @urls = urls
+        super "Redirect loop: #{urls.join(' -> ')}."
+      end
+
+      def url
+        @urls.last
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/errors.rb

@@ -0,0 +1,2 @@
+require_relative 'errors/base_error'
+require_relative 'errors/redirect_loop_error'

data/lib/ruby-link-checker/logger.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+require 'logger'
+
+module LinkChecker
+  class Logger < ::Logger
+    def self.default
+      return @default if @default
+
+      logger = Logger.new(STDOUT)
+      logger.level = Logger::WARN
+      @default = logger
+    end
+  end
+end

data/lib/ruby-link-checker/net/http/checker.rb

@@ -0,0 +1,31 @@
+module LinkChecker
+  module Net
+    module HTTP
+      class Task < ::LinkChecker::Task
+        def run!
+          ::Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+            http.read_timeout = checker.read_timeout if checker.read_timeout
+            http.open_timeout = checker.open_timeout if checker.open_timeout
+            request = ::Net::HTTPGenericRequest.new(method, false, true, uri)
+            request['User-Agent'] = checker.user_agent
+            response = http.request(request)
+            logger.debug "#{method} #{uri}: #{response.code}"
+            result! Result.new(uri, method, original_uri, request, response, options)
+          end
+        end
+      end
+
+      class Checker < LinkChecker::Checker
+        extend ::LinkChecker::Net::HTTP::Config
+        attr_accessor(*LinkChecker::Net::HTTP::Config::ATTRIBUTES)
+
+        def initialize(options = {})
+          LinkChecker::Net::HTTP::Config::ATTRIBUTES.each do |key|
+            send("#{key}=", options[key] || LinkChecker::Net::HTTP::Config.send(key))
+          end
+          super options
+        end
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/net/http/config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Net
+    module HTTP
+      module Config
+        extend self
+
+        ATTRIBUTES = %i[
+          read_timeout
+          open_timeout
+        ].freeze
+
+        attr_accessor(*Config::ATTRIBUTES)
+
+        def reset
+          self.read_timeout = nil
+          self.open_timeout = nil
+        end
+      end
+
+      class << self
+        def configure
+          block_given? ? yield(Config) : Config
+        end
+
+        def config
+          Config
+        end
+      end
+    end
+  end
+end
+
+LinkChecker::Net::HTTP::Config.reset

data/lib/ruby-link-checker/net/http/result.rb

@@ -0,0 +1,53 @@
+module LinkChecker
+  module Net
+    module HTTP
+      class Result < ::LinkChecker::Result
+        attr_accessor :request, :response
+
+        def initialize(uri, method, original_uri, request, response, options)
+          @request = request
+          @response = response
+          super uri, method, original_uri, options
+        end
+
+        def error?
+          false
+        end
+
+        def failure?
+          !success? && !redirect?
+        end
+
+        def code
+          @code ||= begin
+            response.code.to_i
+          rescue StandardError
+            -1
+          end
+        end
+
+        def request_headers
+          request
+        end
+
+        def redirect_to
+          return nil unless response
+
+          response['Location']
+        end
+
+        def redirect?
+          return false unless response
+
+          [301, 302, 303, 307, 308].include?(code)
+        end
+
+        def success?
+          return false unless response
+
+          code >= 200 && code <= 299
+        end
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/net/http.rb

@@ -0,0 +1,3 @@
+require_relative 'http/config'
+require_relative 'http/result'
+require_relative 'http/checker'

data/lib/ruby-link-checker/result.rb

@@ -0,0 +1,74 @@
+module LinkChecker
+  class Result
+    attr_accessor :uri, :result_uri, :method, :options, :checker
+
+    def initialize(current_uri, method, original_uri, options = {})
+      @uri = original_uri
+      @result_uri = current_uri
+      @method = method
+      @options = options
+    end
+
+    def success?
+      false
+    end
+
+    def failure?
+      false
+    end
+
+    def error?
+      false
+    end
+
+    def redirect?
+      false
+    end
+
+    def redirect_to
+      nil
+    end
+
+    def request_headers
+      {}
+    end
+
+    def code
+      nil
+    end
+
+    def error
+      nil
+    end
+
+    def to_s
+      status_s = if success?
+                   'OK'
+                 elsif failure?
+                   'FAIL'
+                 elsif redirect?
+                   'REDIRECT'
+                 else
+                   'ERROR'
+                 end
+      "#{method} #{uri}#{result_uri == uri ? nil : ' (' + result_uri.to_s + ')'}: #{status_s} (#{code})"
+    end
+  end
+
+  class ResultError < Result
+    attr_accessor :error
+
+    def initialize(uri, method, original_uri, error, options = {})
+      @error = error
+      super uri, method, original_uri, options
+    end
+
+    def error?
+      true
+    end
+
+    def code
+      error.class.name
+    end
+  end
+end

data/lib/ruby-link-checker/task.rb

@@ -0,0 +1,20 @@
+module LinkChecker
+  class Task
+    include LinkChecker::Callbacks
+
+    attr_reader :uri, :original_uri, :method, :logger, :options, :checker
+
+    def initialize(checker, uri, method, original_uri, options = {})
+      @checker = checker
+      @logger = checker.logger
+      @uri = uri
+      @original_uri = original_uri || @uri
+      @method = method
+      @options = options
+    end
+
+    def run!
+      raise NotImplementedError
+    end
+  end
+end

data/lib/ruby-link-checker/tasks.rb

@@ -0,0 +1,99 @@
+module LinkChecker
+  class Tasks
+    include LinkChecker::Callbacks
+
+    attr_reader :result, :uri, :original_uri
+
+    def initialize(checker, task_klass, uri, methods, options = {})
+      @uri = uri
+      @retries_left = checker.retries
+      @methods_left = methods.dup
+      @methods = methods.dup
+      @task_klass = task_klass
+      @checker = checker
+      @logger = checker.logger
+      @redirects = [uri]
+      @options = options
+      raise ArgumentError, :tasks_klass unless @task_klass && @task_klass < ::LinkChecker::Task
+    end
+
+    def new_task(uri, method, original_uri, options)
+      task_klass.new(checker, uri, method, original_uri, options)
+    end
+
+    def execute!
+      if retry?
+        @retries_left -= 1
+        retry! @result
+        _queue_task(uri, method, original_uri || uri, options)
+      elsif methods_left.any?
+        @method = methods_left.shift
+        @redirects = [uri]
+        @uri = URI(@uri) unless @uri.is_a?(URI)
+        _queue_task(uri, method, original_uri || uri, options)
+      elsif @result && result.error?
+        error! @result
+      else
+        failure! @result
+      end
+    rescue StandardError => e
+      logger.error("#{self}##{__method__}") { e }
+      _handle_result ResultError.new(uri, method, original_uri || uri, e, options)
+    end
+
+    private
+
+    attr_reader :logger, :methods_left, :options, :task_klass, :redirects, :checker, :method
+
+    def retries
+      checker.retries
+    end
+
+    def first_time?
+      !!method.nil?
+    end
+
+    def retries_left
+      @retries_left ||= retries
+    end
+
+    def retry?
+      !first_time? && retries_left > 0
+    end
+
+    def _queue_task(uri, method, original_uri, options = {})
+      task = new_task(uri, method, original_uri, options)
+      task.on :result do |result|
+        _handle_result result
+      end
+      task.run!
+    rescue StandardError => e
+      logger.error("#{self}##{__method__}") { e }
+      _handle_result ResultError.new(uri, method, original_uri, e, options)
+    end
+
+    def _handle_result(result)
+      @result = result
+      logger.info "#{' ' * (redirects.count - 1)}#{result}"
+      result! result
+      if result.redirect?
+        redirect! result
+        redirected_to_uri = URI.join(uri, result.redirect_to)
+        if redirects.include?(redirected_to_uri)
+          raise LinkChecker::Errors::RedirectLoopError,
+                redirects.push(redirected_to_uri)
+        end
+
+        redirects << redirected_to_uri
+        _queue_task(redirected_to_uri, result.method, uri, options)
+      elsif result.success?
+        success! result
+      else
+        execute!
+      end
+    rescue StandardError => e
+      logger.error("#{self}##{__method__}") { e }
+      _handle_result ResultError.new(result.uri, result.method, result.result_uri, e, options)
+    end
+  end
+end

data/lib/ruby-link-checker/typhoeus/hydra/checker.rb

@@ -0,0 +1,52 @@
+module LinkChecker
+  module Typhoeus
+    module Hydra
+      class Task < ::LinkChecker::Task
+        def run!
+          request = ::Typhoeus::Request.new(
+            uri, {
+              method: method,
+              followlocation: false,
+              timeout: checker.timeout,
+              connecttimeout: checker.connecttimeout,
+              headers: {
+                'User-Agent' => checker.user_agent
+              }
+            }
+          )
+          request.on_complete do |response|
+            if response.timed_out?
+              logger.debug "#{method} #{uri}: #{response.return_code}"
+              result! ResultError.new(uri, method, original_uri, Timeout::Error.new, options)
+            else
+              logger.debug "#{method} #{uri}: #{response.code}"
+              result! Result.new(uri, method, original_uri, request, response, options)
+            end
+          end
+          checker._queue(request)
+        end
+      end
+
+      class Checker < LinkChecker::Checker
+        extend ::LinkChecker::Typhoeus::Hydra::Config
+        attr_accessor(*LinkChecker::Typhoeus::Hydra::Config::ATTRIBUTES)
+
+        def initialize(options = {})
+          LinkChecker::Typhoeus::Hydra::Config::ATTRIBUTES.each do |key|
+            send("#{key}=", options[key] || LinkChecker::Typhoeus::Hydra::Config.send(key))
+          end
+          @hydra = ::Typhoeus::Hydra.new(options[:hydra] || { max_concurrency: 10 })
+          super options
+        end
+
+        def run
+          @hydra.run
+        end
+
+        def _queue(request)
+          @hydra.queue(request)
+        end
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/typhoeus/hydra/config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  module Typhoeus
+    module Hydra
+      module Config
+        extend self
+
+        ATTRIBUTES = %i[
+          timeout
+          connecttimeout
+        ].freeze
+
+        attr_accessor(*Config::ATTRIBUTES)
+
+        def reset
+          self.timeout = 60
+          self.connecttimeout = 10
+        end
+      end
+
+      class << self
+        def configure
+          block_given? ? yield(Config) : Config
+        end
+
+        def config
+          Config
+        end
+      end
+    end
+  end
+end
+
+LinkChecker::Typhoeus::Hydra::Config.reset

data/lib/ruby-link-checker/typhoeus/hydra/result.rb

@@ -0,0 +1,53 @@
+module LinkChecker
+  module Typhoeus
+    module Hydra
+      class Result < ::LinkChecker::Result
+        attr_accessor :request, :response
+
+        def initialize(uri, method, original_uri, request, response, options)
+          @request = request
+          @response = response
+          super uri, method, original_uri, options
+        end
+
+        def error?
+          false
+        end
+
+        def failure?
+          !success? && !redirect? && !error?
+        end
+
+        def code
+          @code ||= begin
+            response.code.to_i
+          rescue StandardError
+            -1
+          end
+        end
+
+        def request_headers
+          request.options[:headers]
+        end
+
+        def redirect_to
+          return nil unless response
+
+          response.headers['Location']
+        end
+
+        def redirect?
+          return false unless response
+
+          [301, 302, 303, 307, 308].include?(code)
+        end
+
+        def success?
+          return false unless response
+
+          code >= 200 && code <= 299
+        end
+      end
+    end
+  end
+end

data/lib/ruby-link-checker/typhoeus/hydra.rb

@@ -0,0 +1,3 @@
+require_relative 'hydra/config'
+require_relative 'hydra/checker'
+require_relative 'hydra/result'

data/lib/ruby-link-checker/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LinkChecker
+  VERSION = '0.1.0'
+end

data/lib/ruby-link-checker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative 'ruby-link-checker/version'
+require_relative 'ruby-link-checker/errors'
+require_relative 'ruby-link-checker/config'
+require_relative 'ruby-link-checker/callbacks'
+require_relative 'ruby-link-checker/logger'
+require_relative 'ruby-link-checker/task'
+require_relative 'ruby-link-checker/tasks'
+require_relative 'ruby-link-checker/checker'
+require_relative 'ruby-link-checker/result'
+require_relative 'ruby-link-checker/net/http'
+require_relative 'ruby-link-checker/typhoeus/hydra'

data/lib/ruby_link_checker.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require 'ruby-link-checker'