ndd-url_checker 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6ffa576d85139cd2300b55ba9c340fdda8608850
+  data.tar.gz: 58835f36306bd29f9375f0f6c65afc7045b7e411
+SHA512:
+  metadata.gz: 84a715175f934072d1b5ec7244302d4c8385d7db3d724d1ac7596e9e98ca00f58bccf12f68c4d88ef4cac4fb865f7ea192f435bcc939783e0dd8a4fcec77281b
+  data.tar.gz: 62ee300dbe403c80a42e8387dc79657549c9bbf0f0be3d3788ed692d2847517bc70ecd460a130be42517d6086b909885b7e47dc29c48c79f72e5ab0eebb80e37

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--colour --format documentation --tag focused

data/.ruby-gemset

@@ -0,0 +1 @@
+ndd-url_checker

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.1.5

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+rvm:
+  - 2.0.0
+  - 2.1.5
+before_install:
+  - gem update --system
+  - gem --version
+branches:
+  only:
+    - master
+    - development
+addons:
+  code_climate:
+    repo_token:
+      secure: "jwblN6fY+HN0u+JfNWKNRNRExOlDH0NbuPprLxqac3kjE3ER11pJieXJKOoeFMcYJ8Z/33qPn4KhOPeYV/0HcZW5YD3G9pjry+c2ha9S7YjIQi8O0BR/iUmqzhvYOT6OoR+khJl9pf5/li3OfXUbAVNdcEDAKARuzl/XnGpqsdE="

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# NDD URL Checker changelog
+
+## Version 0.1.0
+
+* initial commit

data/Gemfile

@@ -0,0 +1,50 @@
+source 'http://rubygems.org'
+
+HOST_OS = RbConfig::CONFIG['host_os']
+
+
+# ------------------------------------------------------------------------------
+# Dependencies required to use the gem.
+gem 'cod',     '~> 0.6'
+gem 'logging', '~> 1.8'
+
+
+# ------------------------------------------------------------------------------
+# Dependencies to develop the gem.
+# Everything needed to run rake, tests, features, etc.
+group :development do
+
+  gem 'bundler',                    '~> 1.7',  require: false
+  gem 'guard',                      '~> 2.8',  require: false
+  gem 'guard-bundler',              '~> 2.0',  require: false
+  gem 'guard-rspec',                '~> 4.3',  require: false
+  gem 'guard-spork',                '~> 2.0',  require: false
+  gem 'jeweler',                    '~> 2.0',  require: false
+  gem 'rdoc',                       '~> 4.1',  require: false
+  gem 'rspec',                      '~> 3.1',  require: false
+  gem 'rspec-collection_matchers',  '~> 1.1',  require: false
+  gem 'simplecov',                  '~> 0.9',  require: false
+  gem 'spork',                      '~> 0.9',  require: false
+  gem 'webmock',                    '~> 1.20', require: false
+  gem 'yard',                       '~> 0.8',  require: false
+
+  case HOST_OS
+    when /darwin/i
+      gem 'growl'
+      gem 'rb-fsevent'
+    when /linux/i
+      gem 'libnotify'
+      gem 'rb-inotify'
+    when /mswin|windows/i
+      gem 'rb-fchange'
+      gem 'rb-notifu'
+      gem 'win32console'
+    else
+      raise "Platform '#{HOST_OS}' is not supported"
+  end
+
+end
+
+group :test do
+  gem 'codeclimate-test-reporter', '~> 0.4', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,152 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.6)
+    builder (3.2.2)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    childprocess (0.5.5)
+      ffi (~> 1.0, >= 1.0.11)
+    cod (0.6.0)
+    codeclimate-test-reporter (0.4.3)
+      simplecov (>= 0.7.1, < 1.0.0)
+    coderay (1.1.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    ffi (1.9.6)
+    formatador (0.2.5)
+    git (1.2.8)
+    github_api (0.12.2)
+      addressable (~> 2.3)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8, < 0.10)
+      hashie (>= 3.3)
+      multi_json (>= 1.7.5, < 2.0)
+      nokogiri (~> 1.6.3)
+      oauth2
+    guard (2.8.2)
+      formatador (>= 0.2.4)
+      listen (~> 2.7)
+      lumberjack (~> 1.0)
+      pry (>= 0.9.12)
+      thor (>= 0.18.1)
+    guard-bundler (2.0.0)
+      bundler (~> 1.0)
+      guard (~> 2.2)
+    guard-compat (0.3.0)
+    guard-rspec (4.4.2)
+      guard (~> 2.1)
+      guard-compat (~> 0.1)
+      rspec (>= 2.99.0, < 4.0)
+    guard-spork (2.0.1)
+      childprocess (>= 0.2.3)
+      guard (~> 2.8.2)
+      spork (>= 0.8.4)
+    hashie (3.3.2)
+    highline (1.6.21)
+    hitimes (1.2.2)
+    jeweler (2.0.1)
+      builder
+      bundler (>= 1.0)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+    json (1.8.1)
+    jwt (1.2.0)
+    libnotify (0.8.4)
+      ffi (>= 1.0.11)
+    listen (2.8.3)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    little-plugger (1.1.3)
+    logging (1.8.2)
+      little-plugger (>= 1.1.3)
+      multi_json (>= 1.8.4)
+    lumberjack (1.0.9)
+    method_source (0.8.2)
+    mini_portile (0.6.1)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.5)
+      mini_portile (~> 0.6.0)
+    oauth2 (1.0.0)
+      faraday (>= 0.8, < 0.10)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.5.2)
+    rake (10.4.2)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    rdoc (4.2.0)
+      json (~> 1.4)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-collection_matchers (1.1.2)
+      rspec-expectations (>= 2.99.0.beta1)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    safe_yaml (1.0.4)
+    simplecov (0.9.1)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    slop (3.6.0)
+    spork (0.9.2)
+    thor (0.19.1)
+    thread_safe (0.3.4)
+    timers (4.0.1)
+      hitimes
+    webmock (1.20.4)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.7)
+  cod (~> 0.6)
+  codeclimate-test-reporter (~> 0.4)
+  guard (~> 2.8)
+  guard-bundler (~> 2.0)
+  guard-rspec (~> 4.3)
+  guard-spork (~> 2.0)
+  jeweler (~> 2.0)
+  libnotify
+  logging (~> 1.8)
+  rb-inotify
+  rdoc (~> 4.1)
+  rspec (~> 3.1)
+  rspec-collection_matchers (~> 1.1)
+  simplecov (~> 0.9)
+  spork (~> 0.9)
+  webmock (~> 1.20)
+  yard (~> 0.8)

data/Guardfile

@@ -0,0 +1,38 @@
+
+# ----------------------------------------------------------------------------------------------------------------------
+# bundler
+# ----------------------------------------------------------------------------------------------------------------------
+
+guard 'bundler' do
+  watch('Gemfile')
+end
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# spork (must be before rspec and cucumber)
+# ----------------------------------------------------------------------------------------------------------------------
+
+guard 'spork',
+      wait: 60 do
+
+  watch('Gemfile')
+  watch('Gemfile.lock')
+
+  # ----- spec directory
+  watch('spec/spec_helper.rb')
+end
+
+
+# ----------------------------------------------------------------------------------------------------------------------
+# rspec
+# ----------------------------------------------------------------------------------------------------------------------
+
+guard :rspec, cmd: 'bundle exec rspec' do
+
+  # ----- lib directory
+  watch(%r{^lib/(.+)\.rb$}) { |m| %W(spec/#{m[1]}_spec.rb) }
+
+  # ----- spec directory
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^spec/support/(.+)\.rb$}) { %W(spec) }
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2014 David DIDIER
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,19 @@
+# NDD URL Checker
+
+[![Build Status](https://secure.travis-ci.org/ddidier/ndd-url_checker.png)](http://travis-ci.org/ddidier/ndd-url_checker)
+[![Dependency Status](https://gemnasium.com/ddidier/ndd-url_checker.png)](https://gemnasium.com/ddidier/ndd-url_checker)
+[![Code Climate](https://codeclimate.com/github/ddidier/ndd-url_checker/badges/gpa.svg)](https://codeclimate.com/github/ddidier/ndd-url_checker)
+[![Test Coverage](https://codeclimate.com/github/ddidier/ndd-url_checker/badges/coverage.svg)](https://codeclimate.com/github/ddidier/ndd-url_checker)
+
+URL validator.
+
+## Prerequisites
+
+This gem is tested with Ruby 2.1.x.
+
+## TODO
+
+## Copyright
+
+Copyright (c) 2014 David DIDIER.
+See `LICENSE.txt` for further details.

data/Rakefile

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification
+  # see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name        = 'ndd-url_checker'
+  gem.summary     = 'Validate URLs'
+  gem.description = 'Validate URLs'
+  gem.homepage    = 'http://github.com/ddidier/ndd-url_checker'
+  gem.license     = 'MIT'
+  gem.email       = 'c_inconnu2@yahoo.fr'
+  gem.authors     = ['David DIDIER']
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+desc 'Code coverage detail'
+task :simplecov do
+  ENV['COVERAGE'] = 'true'
+  Rake::Task['spec'].execute
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "NDD URL Checker #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/lib/ndd/url_checker.rb

@@ -0,0 +1,6 @@
+require 'ndd/url_checker/abstract_url_checker'
+require 'ndd/url_checker/blocking_url_checker'
+require 'ndd/url_checker/forked_url_checker'
+require 'ndd/url_checker/parallel_url_checker'
+require 'ndd/url_checker/status'
+require 'ndd/url_checker/threaded_url_checker'

data/lib/ndd/url_checker/abstract_url_checker.rb

@@ -0,0 +1,30 @@
+require 'ndd/url_checker/status'
+
+module NDD
+  module UrlChecker
+
+    # Abstract class, not very ruby-ish :)
+    # @author David DIDIER
+    class AbstractUrlChecker
+
+      # Checks that the given URL are valid.
+      # If there is only a single URL parameter, returns a NDD::UrlChecker::Status.
+      # If there is only multiple URL parameters, returns a Hash of NDD::UrlChecker::Status indexed by their URI.
+      # @param urls [String|Array<String>] the URLs to check.
+      # @return [NDD::UrlChecker::Status|Hash<String => NDD::UrlChecker::Status>]
+      def check(*urls)
+        raise 'NDD::UrlChecker::UrlChecker#check must be implemented'
+      end
+
+      # Validates that the given URL are valid.
+      # If there is only a single URL parameter, returns a boolean.
+      # If there is only multiple URL parameters, returns a Hash of boolean indexed by their URI.
+      # @param urls [String|Array<String>] the URLs to validate.
+      # @return [NDD::UrlChecker::Status|Hash<String => Boolean>]
+      def validate(*urls)
+        raise 'NDD::UrlChecker::UrlChecker#validate must be implemented'
+      end
+
+    end
+  end
+end

data/lib/ndd/url_checker/blocking_url_checker.rb

@@ -0,0 +1,134 @@
+require 'logging'
+require 'ndd/url_checker/abstract_url_checker'
+require 'ndd/url_checker/status'
+require 'net/http'
+require 'net/https'
+
+module NDD
+  module UrlChecker
+
+    # An URL checker using the blocking Net::HTTP class.
+    # @author David DIDIER
+    class BlockingUrlChecker < AbstractUrlChecker
+
+      # Create a new instance.
+      # @param [Fixnum] maximum_redirects the maximum number of redirects before failing.
+      # @param [Fixnum] timeout the number of seconds to wait before failing.
+      def initialize(maximum_redirects=5, timeout=5)
+        @logger = Logging.logger[self]
+        @maximum_redirects = maximum_redirects
+        @timeout = timeout
+      end
+
+      # Checks that the given URL are valid.
+      # If there is only a single URL parameter, returns a NDD::UrlChecker::Status.
+      # If there is only multiple URL parameters, returns a Hash of NDD::UrlChecker::Status indexed by their URI.
+      # @param [String|Array<String>] urls
+      # @return [NDD::UrlChecker::Status|Hash<String => NDD::UrlChecker::Status>]
+      def check(*urls)
+        @logger.info "Checking #{urls.size} URL(s)"
+        return check_single(urls.first) if urls.size == 1
+        Hash[urls.map { |url| [url, check_single(url)] }]
+      end
+
+      # Validates that the given URL are valid.
+      # If there is only a single URL parameter, returns a boolean.
+      # If there is only multiple URL parameters, returns a Hash of boolean indexed by their URI.
+      # @param [String|Array<String>] urls
+      # @return [NDD::UrlChecker::Status|Hash<String => Boolean>]
+      def validate(*urls)
+        @logger.info "Validating #{urls.size} URL(s)"
+        return validate_single(urls.first) if urls.size == 1
+        Hash[urls.map { |url| [url, validate_single(url)] }]
+      end
+
+
+      private
+
+      # Checks that the given URL is valid.
+      # @param [String] url
+      # @return [NDD::UrlChecker::Status]
+      def check_single(url)
+        begin
+          @logger.debug "Checking: #{url}"
+          status = check_uri(URI.parse(url), Status.new(url))
+        rescue => error
+          status = if unknown_host?(error)
+                     Status.new(url).unknown_host
+                   else
+                     Status.new(url).failed(error)
+                   end
+        end
+        @logger.debug "Checked: #{url} -> #{status.code.upcase}"
+        status
+      end
+
+      # Validates that the given URL are valid.
+      # @param [String] url
+      # @return [Boolean]
+      def validate_single(url)
+        @logger.debug "Validating: #{url}"
+        check_single(url).valid?
+      end
+
+      # Checks that the given URL is valid.
+      # @param [URI::HTTP] uri the URI to check
+      # @param [NDD::UrlChecker::Status] status the current status of the stack
+      # @return [NDD::UrlChecker::Status]
+      def check_uri(uri, status)
+        if status.uris.size() > @maximum_redirects
+          return status.too_many_redirects
+        end
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.read_timeout = @timeout
+        http.use_ssl = true if uri.scheme == 'https'
+        http.start do
+          path = (uri.path.empty?) ? '/' : uri.path
+          http.request_get(path) do |response|
+            case response
+              when Net::HTTPSuccess then
+                return on_success(uri, response, status)
+              when Net::HTTPRedirection then
+                return on_redirection(uri, response, status)
+              else
+                return on_error(uri, response, status)
+            end
+          end
+        end
+      end
+
+      def on_success(uri, response, status)
+        return status if status.redirected?
+        status.direct
+      end
+
+      def on_redirection(uri, response, status)
+        redirected_uri = redirected_uri(uri, response)
+        redirected_status = status.redirected(redirected_uri)
+        check_uri(redirected_uri, redirected_status)
+      end
+
+      def on_error(uri, response, status)
+        status.failed(response)
+      end
+
+      def redirected_uri(uri, response)
+        if response['location'].match(/https?:\/\//)
+          URI(response['location'])
+        else
+          # If the redirect is relative we need to build a new URI using the current URI as a base.
+          URI.join("#{uri.scheme}://#{uri.host}:#{uri.port}", response['location'])
+        end
+      end
+
+      # FIXME: platform dependent?
+      UNKNOWN_HOST_MESSAGE = 'getaddrinfo: Name or service not known'
+
+      def unknown_host?(error)
+        error.is_a?(SocketError) && error.message == UNKNOWN_HOST_MESSAGE
+      end
+
+    end
+  end
+end