twingly-url 1.3.4 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e71d9c7443a57b9f00f9fa78279ea8e7ffd3c2c7
-  data.tar.gz: 0e112dff95a6cd5270edadb8787bfbb51d173fde
+  metadata.gz: 2332efd0ec1df89e43ac8c6dbfe50e32111ee4e1
+  data.tar.gz: f9242f3818d99ef4b49d997898fd22fba951ea9d
 SHA512:
-  metadata.gz: cea28a55da2e632b6dffa00b5a1d8717ade55aaefa9739e2b300340ebc17f05bd93da8d36c7064ab2addc60f088852ba3b802b82b46b84c4a80bfe593202e7b4
-  data.tar.gz: 694eb5dcfaea6fb8601711986aac870d024944b58344d2c351860100f0f870745123bcec0b4773f9487ea3363fade9e759d723b581265433ab04d2672efa6e82
+  metadata.gz: 614aac135b4e9c8fe61dbf9f0e5c13928c5e71f7b552bf980f6ae3331a17eb6cfcbd2d6421d5fdaf49bca9fca723d4a188c1585b8e7b77987a6cd8b27d9abcbc
+  data.tar.gz: 9d33a5f6ebb37eb410779cd40350df8bea3f7ffad72f93e8b8015e0dca086c3a86b66110cde34c28ac05a1da08985aeee2b46eba9fd6e0b4dce6f56e5e01e7f2

data/README.md

@@ -5,10 +5,7 @@
 Twingly URL tools.
 
 * `twingly/url` - Parse and validate URLs
-    * `Twingly::URL.parse` - Returns a Struct with `#url` and `#domain` accessors
-    * `Twingly::URL.validate` - Validates a URL
-* `twingly/url/normalizer` - Normalize URLs
-    * `Twingly::URL::Normalizer.normalize(string)` - Extracts URLs from string (Array)
+    * `Twingly::URL.parse` - Returns one or more `Twingly::URL` instance
 * `twingly/url/hasher` - Generate URL hashes suitable for primary keys
     * `Twingly::URL::Hasher.taskdb_hash(url)` - MD5 hexdigest
     * `Twingly::URL::Hasher.blogstream_hash(url)` - MD5 hexdigest
@@ -16,27 +13,12 @@ Twingly URL tools.
     * `Twingly::URL::Hasher.autopingdb_hash(url)` - SHA256 64-bit signed, native endian digest
     * `Twingly::URL::Hasher.pingloggerdb_hash(url)` - SHA256 64-bit unsigned, native endian digest
 * `twingly/url/utilities` - Utilities to work with URLs
-    * `Twingly::URL::Utilities.remove_scheme(url)` - Removes scheme from HTTP/HTTPS URLs (`http://twingly.com` -> `//twingly.com`)
+    * `Twingly::URL::Utilities.extract_valid_urls` - Returns Array of valid `Twingly::URL`
 
 ## Installation
 
     gem install twingly-url
 
-## Normalization example
-
-```ruby
-require 'twingly/url/normalizer'
-
-Twingly::URL::Normalizer.normalize('http://duh.se')
-# => ["http://www.duh.se/"]
-
-Twingly::URL::Normalizer.normalize('http://duh.se http://blog.twingly.com/')
-# => ["http://www.duh.se/", "http://blog.twingly.com/"]
-
-Twingly::URL::Normalizer.normalize('no URL')
-# => []
-```
-
 ## Tests
 
 Run tests with

data/lib/twingly/url.rb

@@ -1,36 +1,157 @@
-require 'addressable/uri'
-require 'public_suffix'
+require "addressable/uri"
+require "public_suffix"
 
-PublicSuffix::List.private_domains = false
+require_relative "url/null_url"
+require_relative "url/error"
 
-SCHEMES = %w(http https)
+PublicSuffix::List.private_domains = false
 
 module Twingly
-  module URL
-    module_function
+  class URL
+    include Comparable
+
+    SCHEMES = %w(http https)
+    ENDS_WITH_SLASH = /\/+$/
+
+    def self.parse(potential_url)
+      potential_url = String(potential_url)
+      potential_url = potential_url.scrub
+      potential_url = potential_url.strip
+
+      internal_parse(potential_url)
+    rescue Twingly::URL::Error, Twingly::URL::Error::ParseError => error
+      NullURL.new
+    end
+
+    def self.internal_parse(potential_url)
+      if potential_url.is_a?(Addressable::URI)
+        addressable_uri = potential_url
+      else
+        addressable_uri = Addressable::URI.heuristic_parse(potential_url)
+      end
+
+      raise Twingly::Error::ParseError if addressable_uri.nil?
+
+      public_suffix_domain = PublicSuffix.parse(addressable_uri.display_uri.host)
+
+      self.new(addressable_uri, public_suffix_domain)
+    rescue Addressable::URI::InvalidURIError, PublicSuffix::DomainInvalid => error
+      error.extend(Twingly::URL::Error)
+      raise
+    end
+
+    def initialize(addressable_uri, public_suffix_domain)
+      unless addressable_uri.is_a?(Addressable::URI)
+        raise ArgumentError, "First parameter must be an Addressable::URI"
+      end
+
+      unless public_suffix_domain.is_a?(PublicSuffix::Domain)
+        raise ArgumentError, "Second parameter must be a PublicSuffix::Domain"
+      end
+
+      @addressable_uri      = addressable_uri
+      @public_suffix_domain = public_suffix_domain
+    end
+
+    def scheme
+      addressable_uri.scheme
+    end
+
+    def trd
+      public_suffix_domain.trd
+    end
+
+    def sld
+      public_suffix_domain.sld
+    end
+
+    def tld
+      public_suffix_domain.tld
+    end
+
+    def domain
+      public_suffix_domain.domain
+    end
 
-    UrlObject = Struct.new(:url, :domain) do
-      def valid?
-        url && domain && SCHEMES.include?(url.normalized_scheme)
+    def host
+      addressable_uri.host
+    end
+
+    def origin
+      addressable_uri.origin
+    end
+
+    def path
+      addressable_uri.path
+    end
+
+    def without_scheme
+      self.to_s.sub(/\A#{scheme}:/, "")
+    end
+
+    def normalized
+      normalized_url = addressable_uri.dup
+
+      normalized_url.scheme = normalized_scheme
+      normalized_url.host   = normalized_host
+      normalized_url.path   = normalized_path
+
+      self.class.internal_parse(normalized_url)
+    end
+
+    def normalized_scheme
+      addressable_uri.scheme.downcase
+    end
+
+    def normalized_host
+      host   = addressable_uri.normalized_host
+      domain = public_suffix_domain
+
+      unless domain.subdomain?
+        host = "www.#{host}"
       end
+
+      host = normalize_blogspot(host, domain)
+
+      host
+    end
+
+    def normalized_path
+      path = strip_trailing_slashes(addressable_uri.path)
+
+      (path.empty?) ? "/" : path
+    end
+
+    def valid?
+      addressable_uri && public_suffix_domain && SCHEMES.include?(normalized_scheme)
+    end
+
+    def <=>(other)
+      self.to_s <=> other.to_s
     end
 
-    def parse(potential_url)
-      url, domain = extract_url_and_domain(potential_url)
-      UrlObject.new(url, domain)
+    def to_s
+      addressable_uri.to_s
     end
 
-    def extract_url_and_domain(potential_url)
-      url    = Addressable::URI.heuristic_parse(potential_url)
-      domain = PublicSuffix.parse(url.host) if url
+    def inspect
+      sprintf("#<%s:0x%x %s>", self.class.name, __id__, self.to_s)
+    end
 
-      [url, domain]
-    rescue PublicSuffix::DomainInvalid, Addressable::URI::InvalidURIError
-      []
+    private
+
+    attr_reader :addressable_uri, :public_suffix_domain
+
+    def normalize_blogspot(host, domain)
+      if domain.sld.downcase == "blogspot"
+        host.sub(/\Awww\./i, "").sub(/#{domain.tld}\z/i, "com")
+      else
+        host
+      end
     end
 
-    def validate(potential_url)
-      parse(potential_url).valid?
+    def strip_trailing_slashes(path)
+      path.sub(ENDS_WITH_SLASH, "")
     end
   end
 end

data/lib/twingly/url/error.rb

@@ -0,0 +1,8 @@
+module Twingly
+  class URL
+    module Error
+      class ParseError < StandardError
+      end
+    end
+  end
+end

data/lib/twingly/url/hasher.rb

@@ -1,7 +1,7 @@
 require 'digest'
 
 module Twingly
-  module URL
+  class URL
     module Hasher
       module_function
 

data/lib/twingly/url/null_url.rb

@@ -0,0 +1,30 @@
+module Twingly
+  class URL
+    class NullURL
+      include Comparable
+
+      def method_missing(name, *)
+        error = NoMethodError.new("undefined method `#{name}'")
+        raise error unless Twingly::URL.instance_methods.include?(name)
+
+        ""
+      end
+
+      def normalized
+        self
+      end
+
+      def valid?
+        false
+      end
+
+      def <=>(other)
+        self.to_s <=> other.to_s
+      end
+
+      def to_s
+        ""
+      end
+    end
+  end
+end

data/lib/twingly/url/utilities.rb

@@ -1,12 +1,14 @@
 module Twingly
-  module URL
+  class URL
     module Utilities
       module_function
 
-      PROTOCOL_EXPRESSION = /^https?:/i
-
-      def remove_scheme(url)
-        url.sub(PROTOCOL_EXPRESSION, '')
+      def extract_valid_urls(text_or_array)
+        potential_urls = Array(text_or_array).flat_map(&:split)
+        potential_urls.map do |potential_url|
+          url = Twingly::URL.parse(potential_url)
+          url if url.valid?
+        end.compact
       end
     end
   end

data/lib/twingly/version.rb

@@ -0,0 +1,5 @@
+module Twingly
+  class URL
+    VERSION = "2.0.0"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twingly-url
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 2.0.0
 platform: ruby
 authors:
 - Twingly AB
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-04 00:00:00.000000000 Z
+date: 2015-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -38,20 +38,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
-- !ruby/object:Gem::Dependency
-  name: minitest-reporters
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -67,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '10'
 - !ruby/object:Gem::Dependency
-  name: shoulda-context
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
@@ -102,12 +88,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - README.md
-- lib/twingly-url-normalizer.rb
 - lib/twingly/url.rb
+- lib/twingly/url/error.rb
 - lib/twingly/url/hasher.rb
-- lib/twingly/url/normalizer.rb
+- lib/twingly/url/null_url.rb
 - lib/twingly/url/utilities.rb
-- lib/version.rb
+- lib/twingly/version.rb
 homepage: http://github.com/twingly/twingly-url
 licenses:
 - MIT
@@ -128,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Ruby library for URL handling

data/lib/twingly-url-normalizer.rb

@@ -1,2 +0,0 @@
-warn "twingly-url-normalizer will be removed, use twingly/url/normalizer"
-require 'twingly/url/normalizer'

data/lib/twingly/url/normalizer.rb

@@ -1,36 +0,0 @@
-require 'twingly/url'
-
-module Twingly
-  module URL
-    module Normalizer
-      module_function
-
-      def normalize(potential_urls)
-        extract_urls(potential_urls).map do |potential_url|
-          normalize_url(potential_url)
-        end.compact
-      end
-
-      def extract_urls(potential_urls)
-        Array(potential_urls).map(&:split).flatten
-      end
-
-      def normalize_url(potential_url)
-        result = Twingly::URL.parse(potential_url)
-
-        return nil unless result.valid?
-
-        unless result.domain.subdomain?
-          result.url.host = "www.#{result.domain}"
-        end
-
-        if result.url.path.empty?
-          result.url.path = "/"
-        end
-
-        result.url.to_s.downcase
-      end
-    end
-  end
-end
-

data/lib/version.rb

@@ -1,5 +0,0 @@
-module Twingly
-  module URL
-    VERSION = '1.3.4'
-  end
-end