url_canonicalize 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3b018935dfe00ea2df8f9265b439a846141a8b03
+  data.tar.gz: 0e103350c0c91b1ae261dd2996b112ef35aa6790
+SHA512:
+  metadata.gz: e30f6a713c580cb396ccf9b717a32b8b56ff61f8aa8b90ab9fff6981383e7d78a90f4c343ae6cd2cb492aea745601c5c195b2cfd4849d16afe8a01b24d9b5672
+  data.tar.gz: 068cf183a3d2050de607f5d098cdf37defe6dc68ec3210f67c3d8fce045e63d4129bc79b870f42a25755594572558a9e70272aff421a476eb84c76e851784979

data/.gitignore

@@ -0,0 +1,51 @@
+*.local
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.hound.yml

@@ -0,0 +1,4 @@
+---
+ruby:
+  enabled: true
+  config_file: .rubocop.yml

data/.rspec

@@ -0,0 +1,5 @@
+--color
+--require spec_helper
+--profile
+--format progress
+--format RspecJunitFormatter --out <%= ENV.fetch('CIRCLE_TEST_REPORTS', 'tmp') %>/rspec/rspec.xml

data/.rubocop.yml

@@ -0,0 +1,55 @@
+---
+AllCops:
+  Exclude:
+    - '*.gemspec'
+
+Style/TrailingCommaInArguments:
+  EnforcedStyleForMultiline: no_comma
+  Enabled: true
+
+StringLiterals:
+  EnforcedStyle: single_quotes
+  Enabled: true
+
+LineLength:
+  Max: 120
+  Exclude:
+    - 'spec/**/*'
+
+MethodLength:
+  Max: 12
+
+DotPosition:
+  Description: 'Checks the position of the dot in multi-line method calls.'
+  EnforcedStyle: leading
+  Enabled: true
+
+ClassAndModuleChildren:
+  Description: 'Checks style of children classes and modules.'
+  EnforcedStyle: nested
+  Enabled: true
+
+Documentation:
+  Description: 'Document classes and non-namespace modules.'
+  Enabled: false
+
+FileName:
+  Description: 'Use snake_case for source file names.'
+  Enabled: true
+
+Style/SymbolArray:
+  Description: 'Use %i or %I for arrays of symbols.'
+  StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#percent-i'
+  Enabled: false # Only available in Ruby 2.0+
+
+Style/ExtraSpacing:
+  Description: 'Do not use unnecessary spacing.'
+  Enabled: true
+
+Lint/LiteralInInterpolation:
+  Description: 'Avoid interpolating literals in strings'
+  AutoCorrect: true
+
+Metrics/ClassLength:
+  CountComments: false  # count full line comments?
+  Max: 120

data/.ruby-gemset

@@ -0,0 +1 @@
+url_canonicalize

data/.ruby-version

@@ -0,0 +1 @@
+2.1.9

data/Gemfile

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'rspec'
+  gem 'rspec_junit_formatter'
+  gem 'webmock'
+  gem 'simplecov'
+  gem 'coveralls', require: false
+end
+
+local_gemfile = 'Gemfile.local'
+
+if File.exist?(local_gemfile)
+  eval(File.read(local_gemfile)) # rubocop:disable Lint/Eval
+end

data/Gemfile.local.example

@@ -0,0 +1,11 @@
+# Copy this file to a file named Gemfile.local to add these gems to your dev toolset
+# Feel free to modify Gemfile.local to suit your own preferences
+group :development do
+  gem 'rake'
+  gem 'gem-release'
+  gem 'rubocop'
+  gem 'listen', '~> 3.0', '< 3.1' # Dependency of guard, 3.1 requires Ruby 2.2+
+  gem 'guard'
+  gem 'guard-rspec'
+  gem 'guard-rubocop'
+end

data/Gemfile.lock

@@ -0,0 +1,131 @@
+PATH
+  remote: .
+  specs:
+    url_canonicalize (0.0.1)
+      addressable (~> 2)
+      nokogiri (~> 1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.4.0)
+    ast (2.3.0)
+    builder (3.2.2)
+    coderay (1.1.1)
+    coveralls (0.8.15)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.12.0)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.1)
+      tins (>= 1.6.0, < 2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    ffi (1.9.14)
+    formatador (0.2.5)
+    gem-release (0.7.4)
+    guard (2.14.0)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    guard-rubocop (1.2.0)
+      guard (~> 2.0)
+      rubocop (~> 0.20)
+    hashdiff (0.3.0)
+    json (2.0.2)
+    listen (3.0.8)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+    lumberjack (1.0.10)
+    method_source (0.8.2)
+    mini_portile2 (2.1.0)
+    nenv (0.3.0)
+    nokogiri (1.6.8.1)
+      mini_portile2 (~> 2.1.0)
+    notiffany (0.1.1)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parser (2.3.1.4)
+      ast (~> 2.2)
+    powerpack (0.1.1)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rainbow (2.1.0)
+    rake (11.3.0)
+    rb-fsevent (0.9.7)
+    rb-inotify (0.9.7)
+      ffi (>= 0.5.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    rspec_junit_formatter (0.2.3)
+      builder (< 4)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    rubocop (0.44.1)
+      parser (>= 2.3.1.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 1.99.1, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.1)
+    safe_yaml (1.0.4)
+    shellany (0.0.1)
+    simplecov (0.12.0)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    term-ansicolor (1.4.0)
+      tins (~> 1.0)
+    thor (0.19.1)
+    tins (1.12.0)
+    unicode-display_width (1.1.1)
+    webmock (2.1.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  coveralls
+  gem-release
+  guard
+  guard-rspec
+  guard-rubocop
+  listen (~> 3.0, < 3.1)
+  rake
+  rspec
+  rspec_junit_formatter
+  rubocop
+  simplecov
+  url_canonicalize!
+  webmock
+
+BUNDLED WITH
+   1.13.5

data/Guardfile

@@ -0,0 +1,16 @@
+guard :rubocop do
+  watch(/.+\.rb$/)
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end
+
+guard(
+  :rspec,
+  all_after_pass: false,
+  all_on_start: false,
+  cmd: 'NO_SIMPLECOV=true bundle exec rspec --fail-fast --format documentation'
+) do
+  watch(%r{spec/.+_spec\.rb$})
+  watch(%r{lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb') { 'spec' }
+  watch(%r{^spec/support/.+\.rb$}) { 'spec' }
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Xenapto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,32 @@
+# URLCanonicalize
+
+URLCanonicalize is a Ruby gem that finds the canonical version of a URL. It
+provides `canonicalize` methods for the String, URI::HTTP, URI::HTTPS and
+Addressable::URI classes.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'url_canonicalize'
+```
+
+## Usage
+
+```ruby
+'http://www.twitter.com'.canonicalize # => 'https://twitter.com/'
+URI('http://www.twitter.com').canonicalize # => #<URI::HTTP:0x00000008767908 URL:https://twitter.com/>
+Addressable::URI.canonicalize('http://www.twitter.com') # => #<Addressable::URI:0x43c9 URI:https://twitter.com/>
+```
+
+## More Information
+
+URLCanonical follows HTTP redirects and also looks for `rel="canonical"` hints
+in both the HTTP headers and the `<head>` section of the response HTML. The URL
+it returns will be both normalized and canonical. The intention is that
+whatever variant of a URL is supplied the result will always be the same. The
+intended use case is for applications that need to dedupe a list of URLs, for
+instance to check if a new URL is already present in a list. If the list is
+built from canonicalized URLs then the resulting set will have fewer URLs that
+point to the same ultimate resource.

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/circle.yml

@@ -0,0 +1,18 @@
+machine:
+  ruby:
+    version: 2.1.9
+
+dependencies:
+  pre:
+    - echo "export rvm_ignore_gemsets_flag=1" >> ~/.rvmrc
+    - gem install bundler
+  override:
+    - bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs=4 --retry=3 --full-index
+
+test:
+  override:
+    - bundle exec rspec:
+        timeout: 600
+        parallel: true
+        files:
+        - spec/**/*_spec.rb

data/lib/monkey_patches/addressable/uri.rb

@@ -0,0 +1,10 @@
+module Addressable
+  # Patch for Addressable's URI class
+  class URI
+    def self.canonicalize(uri)
+      url = parse(uri).to_s # uri can be anything Addressable::URI can handle
+      canonical_url = URLCanonicalize.canonicalize(url)
+      parse(canonical_url)
+    end
+  end
+end

data/lib/monkey_patches/string.rb

@@ -0,0 +1,21 @@
+# Patch for Ruby's String class
+class String
+  def canonicalize
+    URLCanonicalize.canonicalize(self)
+  end
+
+  def ansi_attributes(*args)
+    "\e[#{args.join(';')}m#{self}\e[0m"
+  end
+
+  colors = %w(black red green yellow blue magenta cyan white)
+
+  colors.each_with_index do |fg_color, i|
+    fg = 30 + i
+    define_method(fg_color) { ansi_attributes(fg) }
+
+    colors.each_with_index do |bg_color, j|
+      define_method("#{fg_color}_on_#{bg_color}") { ansi_attributes(fg, 40 + j) }
+    end
+  end
+end

data/lib/monkey_patches/uri.rb

@@ -0,0 +1,9 @@
+module URI
+  # URI having the HTTP protocol
+  class HTTP
+    def canonicalize
+      new_url = URLCanonicalize.canonicalize(to_s)
+      ::URI.parse(new_url)
+    end
+  end
+end

data/lib/url_canonicalize/exception.rb

@@ -0,0 +1,9 @@
+# Core methods
+module URLCanonicalize
+  # Local exception classes to make handling exceptions easier
+  class Exception < RuntimeError
+    URI = Class.new(self)
+    Redirect = Class.new(self)
+    Failure = Class.new(self)
+  end
+end

data/lib/url_canonicalize/http.rb

@@ -0,0 +1,155 @@
+module URLCanonicalize
+  # Persistent connection for possible repeated requests to the same host
+  class HTTP
+    def fetch
+      loop { break last_known_good if handle_response }
+    end
+
+    def uri
+      @uri ||= URLCanonicalize::URI.parse(url) # Malformed URLs will raise a URLCanonicalize exception
+    end
+
+    def url=(value)
+      @url = value.to_s
+      @uri = nil
+    end
+
+    def request(request_object)
+      http.request request_object
+    end
+
+    private
+
+    attr_reader :last_known_good
+
+    def initialize(raw_url)
+      @raw_url = raw_url
+    end
+
+    # Fetch the response
+    def response
+      @response ||= Request.new(self).fetch
+    end
+
+    # Parse the response, and clear the response ready to follow the next redirect
+    def handle_response
+      result = parse_response
+      @response = nil
+      result
+    end
+
+    # Parse the response
+    def parse_response
+      case response
+      when Net::HTTPSuccess
+        handle_success
+      when URLCanonicalize::Response::Redirect
+        redirect_loop_detected? || max_redirects_reached?
+      when URLCanonicalize::Response::CanonicalFound
+        handle_canonical_found
+      when URLCanonicalize::Response::Failure
+        handle_failure
+      else
+        handle_unhandled_response
+      end
+    end
+
+    def redirect_loop_detected?
+      if redirect_list.include?(response.url)
+        return true if last_known_good
+        raise URLCanonicalize::Exception::Redirect, 'Redirect loop detected'
+      end
+
+      redirect_list << response.url
+      increment_redirects
+      set_url_from_response
+      false
+    end
+
+    def max_redirects_reached?
+      return false unless @redirects > options[:max_redirects]
+      return true if last_known_good
+      raise URLCanonicalize::Exception::Redirect, "#{@redirects} redirects is too many"
+    end
+
+    def redirect_list
+      @redirect_list ||= []
+    end
+
+    def increment_redirects
+      @redirects = redirects + 1
+    end
+
+    def redirects
+      @redirects ||= 0
+    end
+
+    def handle_canonical_found
+      @last_known_good = response.response
+      return true if response.url == url || redirect_list.include?(response.url)
+      set_url_from_response
+      false
+    end
+
+    def set_url_from_response
+      self.url = response.url
+    end
+
+    def handle_failure
+      return true if last_known_good
+      raise URLCanonicalize::Exception::Failure, "#{response.failure_class}: #{response.message}"
+    end
+
+    def handle_unhandled_response
+      raise URLCanonicalize::Exception::Failure, "Unhandled response type: #{response.class}"
+    end
+
+    def handle_success
+      @last_known_good = response
+      true
+    end
+
+    def url
+      @url ||= @raw_url.to_s
+    end
+
+    def http
+      return @http if same_host_and_port # reuse connection
+
+      @previous = uri
+      @http = new_http
+    end
+
+    def same_host_and_port
+      uri.host == previous.host && uri.port == previous.port
+    end
+
+    def previous
+      @previous ||= Struct.new(:host, :port).new
+    end
+
+    def new_http
+      h = Net::HTTP.new uri.host, uri.port
+
+      h.open_timeout = options[:open_timeout]
+      h.read_timeout = options[:read_timeout]
+
+      if uri.scheme == 'https'
+        h.use_ssl = true # Can generate exception
+        h.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      else
+        h.use_ssl = false
+      end
+
+      h
+    end
+
+    def options
+      @options ||= {
+        open_timeout: 8, # Twitter responds in >5s
+        read_timeout: 15,
+        max_redirects: 10
+      }
+    end
+  end
+end

data/lib/url_canonicalize/request.rb

@@ -0,0 +1,144 @@
+module URLCanonicalize
+  # Make an HTTP request
+  class Request
+    def fetch
+      handle_response
+    end
+
+    private
+
+    attr_reader :http, :http_method
+
+    def initialize(http, http_method = :head)
+      @http = http
+      @http_method = http_method
+    end
+
+    def response
+      @response ||= http.request request # Some URLs can throw an exception here
+    end
+
+    def request
+      @request ||= request_for_method
+    end
+
+    def handle_response
+      case response
+      when Net::HTTPSuccess
+        look_for_canonical
+      when Net::HTTPRedirection
+        handle_redirection
+      else
+        handle_failure
+      end
+    rescue *NETWORK_EXCEPTIONS => e
+      handle_failure(e.class, e.message)
+    end
+
+    def look_for_canonical
+      # Look in response Link header
+      if response['link'] =~ /<(?<url>.+)>\s*;\s*rel="canonical"/i
+        URLCanonicalize::Response::CanonicalFound.new($LAST_MATCH_INFO['url'])
+      elsif http_method == :head
+        self.http_method = :get
+        fetch
+      else
+        canonical_url ? URLCanonicalize::Response::CanonicalFound.new(canonical_url, response) : response
+      end
+    end
+
+    def handle_redirection
+      case response
+      when Net::HTTPFound, Net::HTTPMovedTemporarily, Net::HTTPTemporaryRedirect
+        self.http_method = :get
+        look_for_canonical
+      else
+        URLCanonicalize::Response::Redirect.new(response['location'])
+      end
+    end
+
+    def handle_failure(klass = response.class, message = response.message)
+      URLCanonicalize::Response::Failure.new(klass, message)
+    end
+
+    def html
+      @html ||= Nokogiri::HTML response.body
+    end
+
+    def canonical_url_element
+      @canonical_url_element ||= html.xpath('//head/link[@rel="canonical"]').first
+    end
+
+    def canonical_url
+      @canonical_url ||= canonical_url_element['href'] if @canonical_url_element.is_a?(Nokogiri::XML::Element)
+    end
+
+    def uri
+      @uri ||= http.uri
+    end
+
+    def url
+      @url ||= uri.to_s
+    end
+
+    def host
+      @host ||= uri.host
+    end
+
+    def request_for_method
+      r = base_request
+      headers.each { |header_key, header_value| r[header_key] = header_value }
+      r
+    end
+
+    def base_request
+      check_http_method
+
+      case http_method
+      when :head
+        Net::HTTP::Head.new uri
+      when :get
+        Net::HTTP::Get.new uri
+      else
+        raise URLCanonicalize::Exception::Request, "Unknown method: #{method}"
+      end
+    end
+
+    def headers
+      @headers ||= {
+        'Accept-Language' => 'en-US,en;q=0.8',
+        'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; WOW64) '\
+                        'AppleWebKit/537.36 (KHTML, like Gecko) '\
+                        'Chrome/51.0.2704.103 Safari/537.36'
+      }
+    end
+
+    def http_method=(value)
+      @http_method = value
+      @request = nil
+      @response = nil
+    end
+
+    # Some sites treat HEAD requests as suspicious activity and block the
+    # requester after a few attempts. For these sites we'll use GET requests
+    # only
+    def check_http_method
+      @http_method = :get if host =~ /(linkedin|crunchbase).com/
+    end
+
+    NETWORK_EXCEPTIONS = [
+      EOFError,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::EHOSTUNREACH,
+      Errno::EINVAL,
+      Errno::ENETUNREACH,
+      Errno::ETIMEDOUT,
+      Net::OpenTimeout,
+      Net::ReadTimeout,
+      OpenSSL::SSL::SSLError,
+      SocketError,
+      Timeout::Error
+    ].freeze
+  end
+end

data/lib/url_canonicalize/response.rb

@@ -0,0 +1,39 @@
+module URLCanonicalize
+  # The response from an HTTP request
+  module Response
+    class Generic
+      attr_reader :url
+
+      private
+
+      def initialize(url)
+        @url = url
+      end
+    end
+
+    Redirect = Class.new(Generic)
+
+    class CanonicalFound < Generic
+      attr_reader :response
+
+      private
+
+      def initialize(url, response)
+        @url = url
+        @response = response
+      end
+    end
+
+    # It barfed
+    class Failure
+      attr_reader :failure_class, :message
+
+      private
+
+      def initialize(failure_class, message)
+        @failure_class = failure_class
+        @message = message
+      end
+    end
+  end
+end

data/lib/url_canonicalize/uri.rb

@@ -0,0 +1,25 @@
+module URLCanonicalize
+  # Manage the URL into a URI with local exception handling
+  class URI
+    class << self
+      def parse(url)
+        uri = ::URI.parse(url)
+        uri if valid?(uri)
+      rescue ::URI::InvalidURIError => e
+        new_exception = URLCanonicalize::Exception::URI.new("#{e.class}: #{e.message}")
+        new_exception.set_backtrace e.backtrace
+        raise new_exception
+      end
+
+      private
+
+      def valid?(uri)
+        raise URLCanonicalize::Exception::URI, "#{uri} must be http or https" unless VALID_CLASSES.include?(uri.class)
+        raise URLCanonicalize::Exception::URI, "Missing host name in #{uri}" unless uri.host
+        true
+      end
+
+      VALID_CLASSES = [::URI::HTTP, ::URI::HTTPS].freeze
+    end
+  end
+end

data/lib/url_canonicalize/version.rb

@@ -0,0 +1,3 @@
+module URLCanonicalize
+  VERSION = '0.0.1'.freeze
+end

data/lib/url_canonicalize.rb

@@ -0,0 +1,30 @@
+require 'uri'
+require 'addressable/uri'
+require 'net/http'
+require 'nokogiri'
+
+autoload :OpenSSL, 'openssl'
+
+# Core methods
+module URLCanonicalize
+  autoload :Exception, 'url_canonicalize/exception'
+  autoload :HTTP, 'url_canonicalize/http'
+  autoload :Request, 'url_canonicalize/request'
+  autoload :Response, 'url_canonicalize/response'
+  autoload :URI, 'url_canonicalize/uri'
+  autoload :VERSION, 'url_canonicalize/version'
+
+  class << self
+    def canonicalize(url)
+      fetch(url).uri.to_s
+    end
+
+    def fetch(url)
+      URLCanonicalize::HTTP.new(url).fetch
+    end
+  end
+end
+
+require 'monkey_patches/uri'
+require 'monkey_patches/string'
+require 'monkey_patches/addressable/uri'

data/url_canonicalize.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'url_canonicalize/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'url_canonicalize'
+  s.version       = URLCanonicalize::VERSION
+  s.authors       = ['Dominic Sayers']
+  s.email         = ['developers@xenapto.com']
+  s.summary       = 'Finds the canonical version of a URL'
+  s.description   = 'Rubygem that finds the canonical version of a URL by '\
+                    'providing #canonicalize methods for the String, URI::HTTP'\
+                    ', URI::HTTPS and Addressable::URI classes'
+  s.homepage      = 'https://github.com/Xenapto/url_canonicalize'
+  s.license       = 'MIT'
+
+  s.files = `git ls-files`.split($RS).reject do |file|
+    file =~ /^spec\//
+  end
+
+  s.test_files = []
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.add_dependency 'addressable', '~> 2' # To normalize URLs
+  s.add_dependency 'nokogiri', '~> 1' # To look for <link rel="canonical" ...> in HTML
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: url_canonicalize
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Dominic Sayers
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-10-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: 'Rubygem that finds the canonical version of a URL by providing #canonicalize
+  methods for the String, URI::HTTP, URI::HTTPS and Addressable::URI classes'
+email:
+- developers@xenapto.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".hound.yml"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-gemset"
+- ".ruby-version"
+- Gemfile
+- Gemfile.local.example
+- Gemfile.lock
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- circle.yml
+- lib/monkey_patches/addressable/uri.rb
+- lib/monkey_patches/string.rb
+- lib/monkey_patches/uri.rb
+- lib/url_canonicalize.rb
+- lib/url_canonicalize/exception.rb
+- lib/url_canonicalize/http.rb
+- lib/url_canonicalize/request.rb
+- lib/url_canonicalize/response.rb
+- lib/url_canonicalize/uri.rb
+- lib/url_canonicalize/version.rb
+- url_canonicalize.gemspec
+homepage: https://github.com/Xenapto/url_canonicalize
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.7
+signing_key: 
+specification_version: 4
+summary: Finds the canonical version of a URL
+test_files: []