ruby-link-checker 0.1.0

data/spec/ruby-link-checker/typhoeus/hydra/checker_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe LinkChecker::Typhoeus::Hydra::Checker do
+  module TestLinkChecker
+    class Task < LinkChecker::Typhoeus::Hydra::Task; end
+
+    class LinkChecker < LinkChecker::Typhoeus::Hydra::Checker
+      def check(url, options = {})
+        super url, options
+        @hydra.run
+      end
+    end
+  end
+  before :all do
+    VCR.configure do |config|
+      config.hook_into :typhoeus
+    end
+  end
+
+  describe TestLinkChecker::LinkChecker do
+    it_behaves_like 'a link checker'
+
+    context 'with timeout options', vcr: { cassette_name: '200' } do
+      before do
+        LinkChecker::Typhoeus::Hydra.configure do |config|
+          config.timeout = 5
+          config.connecttimeout = 10
+        end
+        expect(Typhoeus::Request).to receive(:new).with(
+          URI(url),
+          hash_including(timeout: 5, connecttimeout: 10)
+        ).and_call_original
+      end
+
+      include_context 'with url'
+
+      it 'creates requests with a default timeout' do
+        expect(result.success?).to be true
+      end
+    end
+
+    context 'timeout', vcr: { cassette_name: '200' } do
+      before do
+        allow_any_instance_of(Typhoeus::Response).to receive(:timed_out?).and_return(true)
+      end
+
+      include_context 'with url'
+
+      it 'times out' do
+        expect(result.success?).to be false
+        expect(result.error?).to be true
+        expect(result.to_s).to eq 'GET https://www.example.org: ERROR (Timeout::Error)'
+      end
+
+      context 'with metadata' do
+        let(:options) { { foo: :bar } }
+
+        it 'times out' do
+          expect(result.error?).to be true
+          expect(result.options).to eq(foo: :bar)
+        end
+      end
+    end
+  end
+end

data/spec/ruby-link-checker/version_spec.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe LinkChecker do
+  it 'has a version' do
+    expect(LinkChecker::VERSION).not_to be_nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'rubygems'
+
+require 'simplecov'
+SimpleCov.start
+
+require 'rspec'
+require 'ruby-link-checker'
+
+RSpec.configure(&:raise_errors_for_deprecations!)
+
+Dir[File.join(File.dirname(__FILE__), 'support', '**/*.rb')].each do |file|
+  require file
+end

data/spec/support/config.rb

@@ -0,0 +1,8 @@
+RSpec.configure do |config|
+  config.before do
+    LinkChecker::Logger.default.level = Logger::DEBUG
+  end
+  config.after do
+    LinkChecker::Config.reset
+  end
+end

data/spec/support/link_checker.rb

@@ -0,0 +1,322 @@
+shared_context 'a link checker' do
+  context 'user-agent' do
+    subject do
+      described_class.new(user_agent: 'user/agent')
+    end
+
+    it 'updates user-agent' do
+      expect(subject.user_agent).to eq 'user/agent'
+    end
+  end
+
+  context 'check' do
+    let(:url) { 'https://www.example.org' }
+
+    include_context 'with result'
+
+    context 'with metadata' do
+      before do
+        subject.check(url, foo: 'bar')
+      end
+
+      context 'GET' do
+        subject do
+          described_class.new(methods: ['GET'])
+        end
+
+        context 'check' do
+          context '200', vcr: { cassette_name: '200' } do
+            it 'passes metadata' do
+              expect(result.options).to eq(foo: 'bar')
+            end
+          end
+
+          context '404', vcr: { cassette_name: '404' } do
+            it 'passes metadata' do
+              expect(result.options).to eq(foo: 'bar')
+            end
+          end
+
+          context 'error', vcr: { cassette_name: '404' } do
+            let(:url) { '\/invalid-url' }
+
+            it 'passes metadata' do
+              expect(result.options).to eq(foo: 'bar')
+            end
+          end
+
+          context 'a redirect loop', vcr: { cassette_name: '301+301' } do
+            it 'passes metadata' do
+              expect(result.options).to eq(foo: 'bar')
+            end
+          end
+        end
+      end
+    end
+
+    context 'without results' do
+      before do
+        subject.check(url, foo: 'bar')
+      end
+
+      context 'GET' do
+        subject do
+          described_class.new(results: false, methods: ['GET'])
+        end
+
+        context 'check' do
+          context 'a valid URI that returns a 200', vcr: { cassette_name: '200' } do
+            it 'passes metadata' do
+              expect(subject.results).to be_nil
+            end
+          end
+        end
+      end
+    end
+
+    context 'without metadata' do
+      before do
+        subject.check(url)
+      end
+
+      context 'GET' do
+        subject do
+          described_class.new(methods: ['GET'])
+        end
+
+        context 'check' do
+          context 'a valid URI that returns a 200', vcr: { cassette_name: '200' } do
+            it 'sets user agent' do
+              expect(result.request_headers['User-Agent']).to eq "Ruby Link Checker/#{LinkChecker::VERSION}"
+            end
+
+            it 'returns all metadata' do
+              expect(result.options).to eq({})
+            end
+
+            it 'returns results' do
+              expect(subject.results).to eq(
+                error: [],
+                failure: [],
+                success: [
+                  result
+                ]
+              )
+            end
+
+            it 'succeeds' do
+              expect(result.success?).to be true
+              expect(result.error?).to be false
+              expect(result.failure?).to be false
+              expect(result.uri).to eq URI(url)
+              expect(subject).to have_received(:called!).with(:result, result)
+              expect(subject).to have_received(:called!).with(:success, result)
+            end
+          end
+
+          context 'a 404', vcr: { cassette_name: '404', allow_playback_repeats: true } do
+            it 'fails' do
+              expect(result.success?).to be false
+              expect(result.error?).to be false
+              expect(result.failure?).to be true
+              expect(result.uri).to eq URI(url)
+              expect(result.response.code.to_i).to eq 404
+              expect(subject).to have_received(:called!).with(:failure, result)
+            end
+
+            context 'with 0 retries' do
+              subject do
+                described_class.new(methods: ['GET'], retries: 0)
+              end
+
+              it 'fails' do
+                expect(result.success?).to be false
+                expect(result.error?).to be false
+                expect(result.failure?).to be true
+                expect(result.uri).to eq URI(url)
+                expect(result.response.code.to_i).to eq 404
+                expect(subject).to have_received(:called!).with(:failure, result).once
+                expect(subject).not_to have_received(:called!).with(:retry, anything)
+              end
+            end
+
+            context 'with 1 retry' do
+              subject do
+                described_class.new(methods: ['GET'], retries: 1)
+              end
+
+              it 'fails' do
+                expect(result.success?).to be false
+                expect(result.error?).to be false
+                expect(result.failure?).to be true
+                expect(result.uri).to eq URI(url)
+                expect(result.response.code.to_i).to eq 404
+                expect(subject).to have_received(:called!).with(:failure, result).once
+                expect(subject).to have_received(:called!).with(:retry, anything).once
+              end
+            end
+
+            context 'with 2 retries' do
+              subject do
+                described_class.new(methods: ['GET'], retries: 2)
+              end
+
+              it 'fails' do
+                expect(result.success?).to be false
+                expect(result.error?).to be false
+                expect(result.failure?).to be true
+                expect(result.uri).to eq URI(url)
+                expect(result.response.code.to_i).to eq 404
+                expect(subject).to have_received(:called!).with(:failure, result).once
+                expect(subject).to have_received(:called!).with(:retry, anything).twice
+              end
+            end
+          end
+
+          context 'a redirect on HEAD followed by a 403', vcr: { cassette_name: '301+403' } do
+            it 'calls redirect callback' do
+              expect(result.success?).to be false
+              expect(result.failure?).to be true
+              expect(subject).to have_received(:called!).with(:redirect, anything)
+              expect(subject).to have_received(:called!).with(:failure, result).once
+            end
+
+            it 'reports its original and result urls' do
+              expect(result.uri.to_s).to eq url
+              expect(result.result_uri.to_s).not_to eq url
+              expect(result.result_uri.to_s).to eq 'https://www.dblock.org/'
+            end
+          end
+
+          context 'a redirect on HEAD followed by a 200', vcr: { cassette_name: '301+200' } do
+            it 'calls redirect callback' do
+              expect(result.success?).to be true
+              expect(result.failure?).to be false
+              expect(result.redirect?).to be false
+              expect(subject).to have_received(:called!).with(:redirect, anything)
+              expect(subject).to have_received(:called!).with(:success, result)
+              expect(subject).not_to have_received(:called!).with(:failure, anything)
+            end
+          end
+
+          context 'a redirect loop', vcr: { cassette_name: '301+301' } do
+            it 'calls redirect callback' do
+              expect(result.success?).to be false
+              expect(result.failure?).to be false
+              expect(result.error?).to be true
+              expect(result.error).to be_a LinkChecker::Errors::RedirectLoopError
+              expect(result.redirect?).to be false
+              expect(subject).to have_received(:called!).with(:redirect, anything).twice
+              expect(subject).to have_received(:called!).with(:error, result)
+              expect(subject).not_to have_received(:called!).with(:failure, result)
+              expect(subject).not_to have_received(:called!).with(:success, result)
+            end
+          end
+
+          context 'a retry on 429', vcr: {
+            cassette_name: '429+200',
+            match_requests_on: [lambda { |_request, recorded_request|
+              @matched ||= []
+              if @matched.size + 1 === recorded_request.headers['Index'].first
+                @matched << recorded_request
+                true
+              else
+                false
+              end
+            }]
+          } do
+            subject do
+              described_class.new(methods: ['GET'], retries: 1)
+            end
+
+            it 'calls a retry callback' do
+              expect(result.success?).to be true
+              expect(result.failure?).to be false
+              expect(result.redirect?).to be false
+              expect(subject).to have_received(:called!).with(:retry, anything)
+              expect(subject).to have_received(:called!).with(:success, result)
+              expect(subject).not_to have_received(:called!).with(:failure, anything)
+              expect(subject).not_to have_received(:called!).with(:error, anything)
+            end
+          end
+
+          context 'an invalid URI' do
+            let(:url) { '\/invalid-url' }
+
+            it 'fails' do
+              expect(result.success?).to be false
+              expect(result.failure?).to be false
+              expect(result.error?).to be true
+              expect(result.uri).to eq url
+              expect(subject).to have_received(:called!).with(:result, result)
+              expect(subject).to have_received(:called!).with(:error, result)
+              expect(subject).not_to have_received(:called!).with(:failure)
+              expect(subject).not_to have_received(:called!).with(:success)
+            end
+          end
+        end
+
+        context 'HEAD,GET' do
+          subject do
+            described_class.new(methods: %w[HEAD GET])
+          end
+
+          context 'a valid URI that fails on HEAD and succeeds on GET', vcr: { cassette_name: '404+200' } do
+            it 'succeeds' do
+              expect(result.success?).to be true
+              expect(result.error?).to be false
+              expect(result.failure?).to be false
+              expect(result.uri).to eq URI(url)
+              expect(subject).to have_received(:called!).with(:success, result)
+              expect(subject).not_to have_received(:called!).with(:failure, result)
+            end
+          end
+
+          context 'a valid URI that fails both on HEAD and GET', vcr: { cassette_name: '404+404' } do
+            it 'fails' do
+              expect(result.success?).to be false
+              expect(result.error?).to be false
+              expect(result.failure?).to be true
+              expect(result.uri).to eq URI(url)
+              expect(result.response.code.to_i).to eq 404
+              expect(subject).to have_received(:called!).with(:failure, result).once
+            end
+          end
+
+          context 'a retry on 429', vcr: {
+            cassette_name: '429+429+200',
+            match_requests_on: [lambda { |request, recorded_request|
+              @matched ||= []
+              if recorded_request.method == request.method && @matched.size + 1 === recorded_request.headers['Index'].first
+                @matched << recorded_request
+                true
+              else
+                false
+              end
+            }]
+          } do
+            subject do
+              described_class.new(methods: %w[HEAD GET], retries: 1)
+            end
+
+            it 'executes HEAD twice, then falls back to GET' do
+              expect(result.success?).to be true
+            end
+          end
+
+          context 'a redirect on HEAD followed by a 400 error succeeds on GET',
+                  vcr: { cassette_name: '301+400+301+200' } do
+            it 'calls redirect callback' do
+              expect(result.success?).to be true
+              expect(result.failure?).to be false
+              expect(result.redirect?).to be false
+              expect(subject).to have_received(:called!).with(:redirect, anything).twice
+              expect(subject).to have_received(:called!).with(:success, result).once
+              expect(subject).not_to have_received(:called!).with(:failure, anything)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/support/vcr.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'vcr'
+require 'webmock/rspec'
+
+VCR.configure do |config|
+  config.cassette_library_dir = 'spec/fixtures'
+  # config.default_cassette_options = { record: :new_episodes }
+  config.configure_rspec_metadata!
+  config.before_record do |i|
+    i.response.body.force_encoding('UTF-8')
+  end
+end

data/spec/support/with_result.rb

@@ -0,0 +1,15 @@
+shared_context 'with result' do
+  before do
+    allow(subject).to receive(:called!)
+    subject.on do |event, *data|
+      subject.called! event, *data
+    end
+    subject.on :result do |result|
+      @result = result
+    end
+  end
+
+  let(:result) do
+    @result
+  end
+end

data/spec/support/with_url.rb

@@ -0,0 +1,14 @@
+shared_context 'with url' do
+  subject do
+    described_class.new(methods: ['GET'])
+  end
+
+  let(:options) { {} }
+  let(:url) { 'https://www.example.org' }
+
+  include_context 'with result'
+
+  before do
+    subject.check(url, options)
+  end
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification
+name: ruby-link-checker
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Daniel Doubrovkine
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-04-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email: dblock@dblock.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTING.md
+- Dangerfile
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- RELEASING.md
+- Rakefile
+- benchmarks/check.rb
+- benchmarks/data/opensearch.org/links.txt
+- benchmarks/data/opensearch.org/small.txt
+- coverage/assets/0.12.3/DataTables-1.10.20/images/sort_asc.png
+- coverage/assets/0.12.3/DataTables-1.10.20/images/sort_asc_disabled.png
+- coverage/assets/0.12.3/DataTables-1.10.20/images/sort_both.png
+- coverage/assets/0.12.3/DataTables-1.10.20/images/sort_desc.png
+- coverage/assets/0.12.3/DataTables-1.10.20/images/sort_desc_disabled.png
+- coverage/assets/0.12.3/application.css
+- coverage/assets/0.12.3/application.js
+- coverage/assets/0.12.3/colorbox/border.png
+- coverage/assets/0.12.3/colorbox/controls.png
+- coverage/assets/0.12.3/colorbox/loading.gif
+- coverage/assets/0.12.3/colorbox/loading_background.png
+- coverage/assets/0.12.3/favicon_green.png
+- coverage/assets/0.12.3/favicon_red.png
+- coverage/assets/0.12.3/favicon_yellow.png
+- coverage/assets/0.12.3/images/ui-bg_flat_0_aaaaaa_40x100.png
+- coverage/assets/0.12.3/images/ui-bg_flat_75_ffffff_40x100.png
+- coverage/assets/0.12.3/images/ui-bg_glass_55_fbf9ee_1x400.png
+- coverage/assets/0.12.3/images/ui-bg_glass_65_ffffff_1x400.png
+- coverage/assets/0.12.3/images/ui-bg_glass_75_dadada_1x400.png
+- coverage/assets/0.12.3/images/ui-bg_glass_75_e6e6e6_1x400.png
+- coverage/assets/0.12.3/images/ui-bg_glass_95_fef1ec_1x400.png
+- coverage/assets/0.12.3/images/ui-bg_highlight-soft_75_cccccc_1x100.png
+- coverage/assets/0.12.3/images/ui-icons_222222_256x240.png
+- coverage/assets/0.12.3/images/ui-icons_2e83ff_256x240.png
+- coverage/assets/0.12.3/images/ui-icons_454545_256x240.png
+- coverage/assets/0.12.3/images/ui-icons_888888_256x240.png
+- coverage/assets/0.12.3/images/ui-icons_cd0a0a_256x240.png
+- coverage/assets/0.12.3/loading.gif
+- coverage/assets/0.12.3/magnify.png
+- coverage/index.html
+- lib/ruby-link-checker.rb
+- lib/ruby-link-checker/callbacks.rb
+- lib/ruby-link-checker/checker.rb
+- lib/ruby-link-checker/config.rb
+- lib/ruby-link-checker/errors.rb
+- lib/ruby-link-checker/errors/base_error.rb
+- lib/ruby-link-checker/errors/redirect_loop_error.rb
+- lib/ruby-link-checker/logger.rb
+- lib/ruby-link-checker/net/http.rb
+- lib/ruby-link-checker/net/http/checker.rb
+- lib/ruby-link-checker/net/http/config.rb
+- lib/ruby-link-checker/net/http/result.rb
+- lib/ruby-link-checker/result.rb
+- lib/ruby-link-checker/task.rb
+- lib/ruby-link-checker/tasks.rb
+- lib/ruby-link-checker/typhoeus/hydra.rb
+- lib/ruby-link-checker/typhoeus/hydra/checker.rb
+- lib/ruby-link-checker/typhoeus/hydra/config.rb
+- lib/ruby-link-checker/typhoeus/hydra/result.rb
+- lib/ruby-link-checker/version.rb
+- lib/ruby_link_checker.rb
+- pkg/ruby-link-checker-0.1.0.gem
+- ruby-link-checker.gemspec
+- spec/fixtures/200.yml
+- spec/fixtures/301_200.yml
+- spec/fixtures/301_301.yml
+- spec/fixtures/301_400_301_200.yml
+- spec/fixtures/301_403.yml
+- spec/fixtures/404.yml
+- spec/fixtures/404_200.yml
+- spec/fixtures/404_404.yml
+- spec/fixtures/429_200.yml
+- spec/fixtures/429_429_200.yml
+- spec/ruby-link-checker/callbacks_spec.rb
+- spec/ruby-link-checker/checker_spec.rb
+- spec/ruby-link-checker/config_spec.rb
+- spec/ruby-link-checker/net/http/checker_spec.rb
+- spec/ruby-link-checker/typhoeus/hydra/checker_spec.rb
+- spec/ruby-link-checker/version_spec.rb
+- spec/spec_helper.rb
+- spec/support/config.rb
+- spec/support/link_checker.rb
+- spec/support/vcr.rb
+- spec/support/with_result.rb
+- spec/support/with_url.rb
+homepage: http://github.com/dblock/ruby-link-checker
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubygems_version: 3.1.6
+signing_key:
+specification_version: 4
+summary: Fast ruby link checker.
+test_files: []