aryll 1.0.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7149f50c4b8078d740e2afd666aadf0f46bf23f2
+  data.tar.gz: 550cf05fba224bea0787e8d5b888bc8cfbab4863
+SHA512:
+  metadata.gz: e5227678705900c94e75e91291f27900745173d174eae5b2a3d5c11785df0e199198cc9edf983b1f7b94e15c1dbecc3e99f4337dfbd9f5060cd3d7ef62d739e5
+  data.tar.gz: a18baced3db9580c7f2f224ea89d9e6b0667d6b040001db0e75960edd7e5a0687e897fc236d7a791ecb4fc2bca24f16768d0ccb11f719c016bf002e4130ae53d

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+rdoc/
+.rvmrc
+coverage/

data/.ruby-gemset

@@ -0,0 +1 @@
+aryll

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+sudo: false
+rvm:
+  - '2.2.2'
+  - '2.3.0'
+script: 'bundle exec rake test'
+
+notifications:
+  email:
+  - cz@aegisnet.de

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem 'simplecov', require: false

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2011, kaupert media gmbh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the original author / copyright holder nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

data/README.md

@@ -0,0 +1,51 @@
+# Aryll [![Build Status](https://travis-ci.org/kaupertmedia/kauperts_link_checker.svg?branch=master)](https://travis-ci.org/kaupertmedia/kauperts_link_checker)
+
+**Aryll** is a simple library to check for the well-being of URLs. It supports HTTPS and IDN URIs.
+
+## Installation
+
+Add this line to your application's Gemfile:
+```
+ gem 'aryll'
+```
+
+And then execute:
+```
+ $ bundle
+```
+
+Or install it yourself as:
+```
+ $ gem install aryll
+```
+
+## Usage
+It will check any object that responds to `url`:
+```ruby
+  status = Aryll.check!(my_url)
+  unless status.ok?
+    puts status
+  end
+```
+
+You can ignore 301 permanent redirect that only add a trailing slash like this:
+```ruby
+  status = Aryll::LinkChecker.check!(url, ignore_trailing_slash_redirects: true)
+  unless status.ok?
+    # A redirect from http://example.com/foo to http://example.com/foo/ will be considered ok
+  end
+```
+
+## I18n
+The following keys are used to translate error messages using the I18n gem:
+
+* `kauperts.link_checker.errors.timeout`: message when rescueing from Timeout::Error
+* `kauperts.link_checker.errors.generic_network`: message when (currently) rescueing from all other exceptions
+
+## Credits
+**Aryll** is extracted from a maintenance task made for
+[berlin.kauperts.de](https://berlin.kauperts.de) by [kaupert media gmbh](http://kaupertmedia.de).
+
+## License
+**Aryll** is released under a 3-clause BSD-licence. See the LICENSE file for details.
+

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rubygems'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+require 'rake/testtask'
+require 'rdoc/task'
+
+desc "Default: run unit tests."
+task :default => :test
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Aryll'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/aryll.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aryll/version'
+
+Gem::Specification.new do |s|
+  s.name          = "aryll"
+  s.version       = Aryll::VERSION
+  s.platform      = Gem::Platform::RUBY
+  s.authors       = ["Wolfgang Vogl", "Carsten Zimmermann", "Matthias Viehweger"]
+  s.email         = ["cz@aegisnet.de"]
+  s.homepage      = ""
+  s.summary       = "A simple library to check for the well-being of an URL"
+  s.description   = "A simple library to check for the well-being of an URL"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+  s.required_ruby_version = '>= 2.2'
+
+  s.add_dependency "i18n"
+  s.add_dependency "simpleidn"
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rdoc'
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'minitest', '>= 5.0'
+  s.add_development_dependency 'webmock'
+end
+

data/lib/aryll/international_uri.rb

@@ -0,0 +1,29 @@
+module Aryll
+  class LinkChecker
+
+    class InternationalURI < Struct.new(:url)
+      def domain
+        @domain ||= url_without_protocol.split('/', 2)[0]
+      end
+
+      def idn_domain
+        @idn_domain ||= SimpleIDN.to_ascii domain
+      end
+
+      def to_uri
+        URI.parse(url.gsub(domain, idn_domain))
+      end
+
+      private
+
+      def url_without_protocol
+        @url_without_protocol ||= /^http[s]?:\/\/(.+)/.match(url)[1]
+      end
+    end
+
+    def InternationalURI(url)
+      InternationalURI.new(url).to_uri
+    end
+
+  end
+end

data/lib/aryll/link_checker.rb

@@ -0,0 +1,89 @@
+module Aryll
+  # Checks the status of a web address. The returned status can be accessed via
+  # +status+. It contains either a string representation of a numeric http
+  # status code or an error message.
+  #
+  # Supports HTTPS and IDN-domains.
+  #
+  #
+  # The following keys are used to translate error messages using the I18n gem:
+  # * <tt>kauperts.link_checker.errors.timeout</tt>: rescues from Timeout::Error
+  # * <tt>kauperts.link_checker.errors.generic_network</tt>: (currently) rescues from all other exceptions
+  # * <tt>kauperts.link_checker.status.redirect_permanently</tt>: translation for 301 permanent redirects
+  class LinkChecker
+
+    class << self
+      attr_accessor :ignore_trailing_slash_redirects, :ignore_302_redirects
+
+      def configure
+        yield self
+      end
+
+    end
+
+    attr_reader :url, :status, :ignore_trailing_slash_redirects, :ignore_302_redirects,
+      :open_timeout, :read_timeout, :user_agent
+
+    # === Parameters
+    # * +url+: URL, complete with protocol scheme
+    # * +options+: optional configuration parameters, see below.
+    #
+    # === Available Options
+    # * +ignore_trailing_slash_redirects+: ignores redirects to the same URI but only with an added trailing slash (default: false)
+    # * +ignore_302_redirects+: ignores temporary redirects (default: false)
+    # * +open_timeout+: Passed to Net::HTTP#open_timeout
+    # * +read_timeout+: Passed to Net::HTTP#read_timeout
+    def initialize(url, ignore_trailing_slash_redirects: false, ignore_302_redirects: false, open_timeout: 5, read_timeout: 10, user_agent: "Aryll Spider v#{Aryll::VERSION} (https://github.com/kaupertmedia/aryll)".freeze)
+      @url = url
+
+      @ignore_trailing_slash_redirects = ignore_trailing_slash_redirects || self.class.ignore_trailing_slash_redirects
+      @ignore_302_redirects = ignore_302_redirects || self.class.ignore_302_redirects
+      @open_timeout, @read_timeout = open_timeout, read_timeout
+      @user_agent = user_agent
+    end
+
+    # Checks the associated url. Sets and returns +status+
+    def check!
+      @status = begin
+                  if response.code == '301'
+                    @redirect_with_trailing_slash_only = "#{uri}/" == response['location']
+                    StatusMessage.moved_permanently response['location']
+                  else
+                    response.code
+                  end
+                rescue Timeout::Error => e
+                  StatusMessage.timeout e.message
+                rescue Exception => e
+                  StatusMessage.generic e.message
+                end
+    end
+
+    # Returns if a check has been run and the return code was '200 OK'
+    # or if a 301 permanent redirect only added a trailing slash
+    # while +ignore_trailing_slash_redirects+ has been set to true
+    def ok?
+      (status == '200') ||
+        (status == '302' && ignore_302_redirects) ||
+        [@redirect_with_trailing_slash_only, ignore_trailing_slash_redirects].all?
+    end
+
+    def uri
+      @uri ||= InternationalURI(url)
+    end
+
+    private
+
+    def response
+      @response ||= begin
+                      https_opts = { use_ssl: uri.scheme == 'https', verify_mode: OpenSSL::SSL::VERIFY_NONE }
+                      Net::HTTP.start(uri.host, uri.port, https_opts) do |http|
+                        request = Net::HTTP::Get.new uri, 'User-Agent' => user_agent
+                        http.open_timeout = open_timeout
+                        http.read_timeout = read_timeout
+                        http.request request
+                      end
+                    end
+    end
+
+  end
+end

data/lib/aryll/status_message.rb

@@ -0,0 +1,36 @@
+require 'i18n'
+
+module Aryll
+  class LinkChecker
+
+    class StatusMessage
+
+      class << self
+
+        def moved_permanently(message)
+          t 'status.redirect_permanently', 'Moved permanently', message
+        end
+
+        def generic(message)
+          t 'errors.generic_network', 'Generic network error', message
+        end
+
+        def timeout(message)
+          t 'errors.timeout', 'Timeout', message
+        end
+
+        private
+
+        def t(key, default, message)
+          [
+            I18n.t("kauperts.link_checker.#{key}", default: default),
+            "(#{message})"
+          ].join ' '
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/aryll/version.rb

@@ -0,0 +1,3 @@
+module Aryll
+  VERSION = '1.0.0.rc1'
+end

data/lib/aryll.rb

@@ -0,0 +1,18 @@
+require "net/https"
+require "simpleidn"
+require 'aryll/link_checker'
+require 'aryll/international_uri'
+require 'aryll/status_message'
+
+module Aryll
+
+  # Immediately checks +url+ and returns the LinkChecker instance
+  def check!(url, **options)
+    checker = LinkChecker.new(url, **options)
+    checker.check!
+    checker
+  end
+
+  module_function :check!
+
+end

data/test/aryll_test.rb

@@ -0,0 +1,30 @@
+require 'test_helper'
+
+describe Aryll do
+
+  let(:url) do
+    'http://www.example.com'
+  end
+
+  describe '.check!' do
+    it { described_class.method(:check!).arity.must_equal(-2) }
+
+    let(:request_stub) { stub_net_http! }
+
+    before { request_stub }
+
+    it 'returns a link checker instance' do
+      subject = described_class.check!(url)
+      subject.must_be_instance_of Aryll::LinkChecker
+      assert_requested request_stub
+    end
+  end
+
+  def described_class
+    Aryll
+  end
+
+  def stub_net_http!(return_code = "200", host: 'www.example.com', path: '/', protocol: 'http')
+    stub_request(:get, "#{protocol}://#{host}#{path}").to_return(status: return_code.to_i)
+  end
+end

data/test/link_checker_test.rb

@@ -0,0 +1,209 @@
+require 'test_helper'
+
+describe Aryll::LinkChecker do
+
+  let(:url) do
+    'http://www.example.com'
+  end
+
+  subject { described_class.new url }
+
+  let(:translation) { {} }
+
+  before do
+    I18n.backend.store_translations I18n.locale, translation
+  end
+
+  after { I18n.backend.reload! }
+
+  describe 'the class-level configuration defaults' do
+
+    [:ignore_trailing_slash_redirects, :ignore_302_redirects].each do |configuration|
+      describe ".#{configuration}" do
+        it "has a getter and setter" do
+          described_class.must_respond_to configuration
+          described_class.must_respond_to "#{configuration}="
+        end
+      end
+    end
+
+    describe '.configure' do
+      [:ignore_trailing_slash_redirects, :ignore_302_redirects].each do |configuration|
+        it "changes the value for #{configuration}" do
+          config_val = described_class.send configuration
+          proc {
+            described_class.configure do |config|
+              config.send "#{configuration}=", 'foo'
+            end
+          }.must_change -> { described_class.send(configuration) }
+          described_class.send "#{configuration}=", config_val
+        end
+      end
+    end
+
+  end
+
+  describe 'its constructor' do
+    it 'sets the url object' do
+      subject.url.must_equal url
+    end
+  end
+
+  describe '#check!' do
+    it "returns a '200' status" do
+      stub_net_http!
+      subject.check!.must_equal '200'
+    end
+
+    it "returns a '404' status" do
+      stub_net_http!("404")
+      subject.check!.must_equal '404'
+    end
+
+    describe 'with SSL' do
+      let(:url) do
+        'https://www.example.com/'
+      end
+
+      it "returns a '200' status" do
+        stub_net_http! protocol: 'https'
+        subject.check!.must_equal '200'
+      end
+    end
+
+    describe 'with configuration options' do
+      describe 'for trailing slashes' do
+        before { stub_net_http_redirect!("301", location: url + '/') }
+
+        it 'considers trailing slashes for redirects not ok by default' do
+          subject.check!
+          subject.ok?.must_equal false
+        end
+
+        it 'ignores permanent redirects with trailing slash' do
+          subject = described_class.new(url, ignore_trailing_slash_redirects: true )
+          subject.check!
+          subject.ok?.must_equal true
+        end
+      end
+
+      describe 'for temporary redirects (302)' do
+        before do
+          stub_net_http_redirect!("302")
+        end
+
+        it 'considers temporary redirects not ok by default' do
+          subject.check!
+          subject.ok?.must_equal false
+        end
+
+        it 'ignores temporary redirects' do
+          subject = described_class.new(url, ignore_302_redirects: true)
+          subject.check!
+          subject.ok?.must_equal true
+        end
+      end
+    end
+
+    describe 'with timeout errors' do
+      before do
+        stub_request(:any, 'www.example.com').to_timeout
+      end
+
+      it 'handles timeouts' do
+        subject.check!
+        subject.status.must_match(/Timeout .+/)
+      end
+
+      describe 'with I18n' do
+        let(:translation) do
+          { kauperts: { link_checker: { errors: { timeout: "Zeitüberschreitung" } } } }
+        end
+
+        it 'translates the error message' do
+          subject.check!
+          subject.status.must_match(/Zeitüberschreitung .+/)
+        end
+      end
+    end
+
+    describe 'with a unrecognized network error' do
+      let(:generic_error) { Class.new(StandardError) }
+
+      before do
+        stub_request(:any, 'www.example.com').to_raise generic_error
+      end
+
+      it 'handles generic network problem' do
+        subject.check!
+        subject.status.must_match(/Generic network error .+/)
+      end
+
+      describe 'with I18n' do
+        let(:translation) do
+          { kauperts: { link_checker: { errors: { generic_network: "Netzwerkfehler" } } } }
+        end
+
+        it 'translates the error message' do
+          subject.check!
+          subject.status.must_match(/Netzwerkfehler .+/)
+        end
+      end
+    end
+
+    describe 'with IDN domains' do
+      let(:url) do
+        'http://www.trotzköpfchen.de'
+      end
+
+      before do
+        stub_net_http!(host: 'www.xn--trotzkpfchen-9ib.de')
+      end
+
+      it 'handles domain with umlauts' do
+        subject.check!.must_equal '200'
+        subject.ok?.must_equal true
+      end
+    end
+
+  end
+
+  describe '#status' do
+    describe 'with a permanent redirect' do
+      before do
+        stub_net_http_redirect!("301")
+        subject.check!
+      end
+
+      it 'returns the redirection url' do
+        subject.status.must_equal 'Moved permanently (http://auenland.de)'
+      end
+
+      describe 'with I18n' do
+        let(:translation) do
+          { kauperts: { link_checker: { status: { redirect_permanently: "Umgezogen" } } } }
+        end
+
+        it 'translates the status' do
+          subject.status.must_match(/Umgezogen \(http:.+/)
+        end
+      end
+    end
+
+  end
+
+  def described_class
+    Aryll::LinkChecker
+  end
+
+  def stub_net_http!(return_code = "200", host: 'www.example.com', path: '/', protocol: 'http')
+    stub_request(:get, "#{protocol}://#{host}#{path}").to_return(status: return_code.to_i)
+  end
+
+  def stub_net_http_redirect!(return_code = '301', location: "http://auenland.de")
+    stub_request(:get, "http://www.example.com").to_return(
+      status: return_code.to_i,
+      headers: { 'Location' => location }
+    )
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,31 @@
+if ENV['COVERAGE']
+  require 'simplecov'
+  SimpleCov.start
+end
+
+require 'bundler/setup'
+
+require 'minitest/autorun'
+require 'minitest/pride'
+require 'webmock/minitest'
+
+require 'aryll'
+
+I18n.available_locales = :en
+
+module MiniTest
+  module Assertions
+
+    def assert_change(change_proc)
+      before = change_proc.call
+      yield
+      after  = change_proc.call
+      refute_equal before, after
+    end
+
+  end
+
+  module Expectations
+    infect_an_assertion :assert_change, :must_change, :block
+  end
+end

metadata

@@ -0,0 +1,163 @@
+--- !ruby/object:Gem::Specification
+name: aryll
+version: !ruby/object:Gem::Version
+  version: 1.0.0.rc1
+platform: ruby
+authors:
+- Wolfgang Vogl
+- Carsten Zimmermann
+- Matthias Viehweger
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simpleidn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple library to check for the well-being of an URL
+email:
+- cz@aegisnet.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- aryll.gemspec
+- lib/aryll.rb
+- lib/aryll/international_uri.rb
+- lib/aryll/link_checker.rb
+- lib/aryll/status_message.rb
+- lib/aryll/version.rb
+- test/aryll_test.rb
+- test/link_checker_test.rb
+- test/test_helper.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A simple library to check for the well-being of an URL
+test_files:
+- test/aryll_test.rb
+- test/link_checker_test.rb
+- test/test_helper.rb