url_parser 0.4.0 → 0.5.0

data/lib/url_parser/domain.rb

@@ -0,0 +1,102 @@
+require 'ostruct'
+require 'forwardable'
+require 'public_suffix'
+
+module UrlParser
+  class Domain
+    extend Forwardable
+
+    VALID_LABEL = /^(?!\-)[a-z0-9\-]*(?!\-)$/i
+
+    SUFFIX_DEFAULTS = {
+      subdomain: nil,
+      domain: nil,
+      tld: nil,
+      sld: nil,
+      trd: nil,
+      to_s: ''
+    }
+
+    attr_reader :original, :name
+
+    attr_accessor :errors
+
+    def_delegators :suffix, *SUFFIX_DEFAULTS.keys
+
+    def initialize(name, options = {})
+      @original   = name.to_s.downcase.chomp('.')
+      @name       = normalize
+      @errors     = []
+      @validated  = false
+    end
+
+    def labels
+      PublicSuffix::Domain.domain_to_labels(name)
+    end
+
+    def suffix
+      @suffix = begin
+        PublicSuffix.parse(name)
+      rescue
+        self.errors << "'#{original}' is not a valid domain"
+        OpenStruct.new(SUFFIX_DEFAULTS).tap do |os|
+          os.instance_eval('undef to_s')
+        end
+      end
+    end
+
+    def valid?
+      validate unless @validated
+      errors.empty?
+    end
+
+    private
+
+    def normalize
+      Addressable::IDNA.to_ascii(original)
+    end
+
+    def validate
+      validate_labels
+      validate_label_length
+      validate_label_format
+      validate_total_length
+      validate_suffix
+
+      @validated = true
+    end
+
+    def validate_labels
+      if labels.count > 127
+        self.errors << "exceeds 127 labels"
+      end
+    end
+
+    # http://tools.ietf.org/html/rfc1034#section-3.1
+    #
+    def validate_label_length
+      if labels.max_by(&:length).length > 63
+        self.errors << "exceeds maximum label length of 63 characters"
+      end unless labels.empty?
+    end
+
+    def validate_label_format
+      if labels.any? { |label| !(label =~ VALID_LABEL) }
+        self.errors << "contains invalid characters"
+      end
+    end
+
+    # https://blogs.msdn.microsoft.com/oldnewthing/20120412-00/?p=7873/
+    #
+    def validate_total_length
+      if name.length > 253
+        self.errors << "exceeds 253 ASCII characters"
+      end
+    end
+
+    def validate_suffix
+      suffix
+    end
+
+  end
+end

data/lib/url_parser/model.rb

@@ -0,0 +1,233 @@
+module UrlParser
+  class Model
+
+    attr_reader :parsed_uri, :parsed_domain
+
+    def initialize(uri, domain = nil)
+      unless uri.is_a?(Addressable::URI)
+        raise RequiresAddressableURI,
+          "#{uri} must be an Addressable::URI"
+      end
+
+      unless domain.is_a?(UrlParser::Domain)
+        raise RequiresUrlParserDomain,
+          "#{domain} must be a UrlParser::Domain"
+      end if domain
+
+      @parsed_uri     = uri
+      @parsed_domain  = domain || UrlParser::Domain.new(uri.hostname)
+    end
+
+    # Top level URI naming structure / protocol.
+    #
+    def scheme
+      parsed_uri.scheme
+    end
+
+    # Username portion of the userinfo.
+    #
+    def username
+      parsed_uri.user
+    end
+    alias_method :user, :username
+
+    # Password portion of the userinfo.
+    #
+    def password
+      parsed_uri.password
+    end
+
+    # URI username and password for authentication.
+    #
+    def userinfo
+      parsed_uri.userinfo
+    end
+
+    # Fully qualified domain name or IP address.
+    #
+    def hostname
+      parsed_uri.host
+    end
+
+    # Fully qualified domain name or IP address without ww? prefix.
+    #
+    def naked_hostname
+      if www
+        hostname.sub(/\A#{www}./, '')
+      else
+        hostname
+      end
+    end
+
+    # Port number.
+    #
+    def port
+      parsed_uri.port
+    end
+
+    # Hostname and port.
+    #
+    def host
+      result = [ hostname, port ].compact.join(':')
+      result.empty? ? nil : result
+    end
+
+    # The ww? portion of the subdomain.
+    #
+    def www
+      trd.split('.').first.to_s[/www?\d*/] if trd
+    end
+
+    # Returns the top level domain portion, aka the extension.
+    #
+    def tld
+      parsed_domain.tld
+    end
+    alias_method :top_level_domain, :tld
+    alias_method :extension, :tld
+
+    # Returns the second level domain portion, aka the domain part.
+    #
+    def sld
+      parsed_domain.sld
+    end
+    alias_method :second_level_domain, :sld
+    alias_method :domain_name, :sld
+
+    # Returns the third level domain portion, aka the subdomain part.
+    #
+    def trd
+      parsed_domain.trd
+    end
+    alias_method :third_level_domain, :trd
+    alias_method :subdomains, :trd
+
+    # Any non-ww? subdomains.
+    #
+    def naked_trd
+      (trd && www) ? trd[/(?<=^#{www}\.).+/] : trd
+    end
+    alias_method :naked_subdomain, :naked_trd
+
+    # The domain name with the tld.
+    #
+    def domain
+      parsed_domain.domain
+    end
+
+    # All subdomains, include ww?.
+    #
+    def subdomain
+      parsed_domain.subdomain
+    end
+
+    # Scheme and host.
+    #
+    def origin
+      original_origin = parsed_uri.origin
+      original_origin == "null" ? nil : original_origin
+    end
+
+    # Userinfo and host.
+    #
+    def authority
+      parsed_uri.authority
+    end
+
+    # Scheme, userinfo, and host.
+    #
+    def site
+      parsed_uri.site
+    end
+
+    # Directory and segment.
+    #
+    def path
+      parsed_uri.path
+    end
+
+    # Last portion of the path.
+    #
+    def segment
+      (path =~ /\/\z/ ? nil : path.split('/').last) if path
+    end
+
+    # Any directories following the site within the URI.
+    #
+    def directory
+      unless path.nil? || path.empty?
+        parts = path.split('/')
+        if parts.empty?
+          '/'
+        else
+          parts.pop unless segment.to_s.empty?
+          parts.unshift('') unless parts.first.to_s.empty?
+          parts.compact.join('/')
+        end
+      end
+    end
+
+    # Segment if a file extension is present.
+    #
+    def filename
+      segment.to_s[/.+\..+/]
+    end
+
+    # The file extension of the filename.
+    #
+    def suffix
+      if path
+        ext = File.extname(path)
+        ext[0] = '' if ext[0] == '.'
+        ext.empty? ? nil : ext
+      end
+    end
+
+    # Params and values as a string.
+    #
+    def query
+      parsed_uri.query
+    end
+
+    # A hash of params and values.
+    #
+    def query_values
+      parsed_uri.query_values.to_h
+    end
+
+    # Fragment identifier.
+    #
+    def fragment
+      parsed_uri.fragment
+    end
+
+    # Path, query, and fragment.
+    #
+    def resource
+      name = [ segment, query_string, fragment_string ].compact.join
+      name.empty? ? nil : name
+    end
+
+    # Directory and resource - everything after the site.
+    #
+    def location
+      if directory == '/'
+        directory + resource.to_s
+      else
+        result = [ directory, resource ].compact.join('/')
+        result.empty? ? nil : result
+      end
+    end
+
+    private
+
+    def query_string
+      query ? "?#{query}" : nil
+    end
+
+    def fragment_string
+      fragment ? "##{fragment}" : nil
+    end
+
+  end
+end

data/lib/url_parser/option_setter.rb

@@ -0,0 +1,47 @@
+module UrlParser
+  class OptionSetter
+
+    attr_reader :blk
+
+    attr_accessor :options
+
+    def initialize(options = {}, &blk)
+      @options  = options
+      @blk      = blk
+    end
+
+    def unescape!
+      options[:unescape] = true
+    end
+
+    def unembed!
+      options[:unembed] = true
+    end
+
+    def canonicalize!
+      options[:canonicalize] = true
+    end
+
+    def normalize!
+      options[:normalize] = true
+    end
+
+    def clean!
+      unescape!
+      unembed!
+      canonicalize!
+      normalize!
+    end
+
+    def to_hash
+      blk.call(self) if blk
+      self.options
+    end
+    alias_method :to_h, :to_hash
+
+    def method_missing(*args)
+      # no-op
+    end
+
+  end
+end

data/lib/url_parser/parser.rb

@@ -0,0 +1,206 @@
+require 'addressable/uri'
+require 'digest/sha1'
+
+module UrlParser
+  class Parser
+
+    class << self
+
+      def call(uri, options = {}, &blk)
+        return nil unless uri
+
+        parser = new(uri, options).tap do |uri|
+          if block_given?
+            yield uri
+          else
+            uri.parse!
+          end
+        end
+
+        parser.raw? ? parser.raw : parser.uri
+      end
+      alias_method :parse, :call
+
+    end
+
+    attr_reader \
+      :uri,
+      :domain,
+      :default_scheme,
+      :scheme_map,
+      :options
+
+    def initialize(uri, options = {})
+      @uri              = uri
+      @domain           = nil
+      @base_uri         = options.delete(:base_uri) { nil }
+      @default_scheme   = options.delete(:default_scheme) {
+                            UrlParser.configuration.default_scheme
+                          }
+      @embedded_params  = options.delete(:embedded_params) {
+                            UrlParser.configuration.embedded_params
+                          }
+      @scheme_map       = options.delete(:scheme_map) {
+                            UrlParser.configuration.scheme_map
+                          }
+      @raw              = options.delete(:raw) { false }
+      @options          = options
+    end
+
+    def base_uri
+      (@base_uri ? @base_uri : uri).to_s
+    end
+
+    def embedded_params
+      UrlParser.wrap(@embedded_params)
+    end
+
+    def raw?
+      !!@raw
+    end
+
+    def set_default_scheme?
+      !!@default_scheme
+    end
+
+    def unescape
+      UrlParser.unescape(uri)
+    end
+
+    def unescape!
+      @uri = unescape
+    end
+
+    def parse
+      return uri if uri.is_a?(Addressable::URI)
+
+      Addressable::URI.parse(base_uri).tap do |parsed_uri|
+        parsed_uri.join!(uri) if @base_uri
+
+        if options[:host]
+          parsed_uri.host = options[:host]
+        else
+          parts     = parsed_uri.path.to_s.split(/[\/:]/)
+          hostname  = parsed_uri.host || parts.first
+          @domain   = UrlParser::Domain.new(hostname)
+          if @domain.valid?
+            parsed_uri.path = '/' +
+              parts.drop(1).join('/') +
+              parsed_uri.path[/(?<=\/).*(\/)\s*$/, 1].to_s
+            parsed_uri.host = @domain.name
+          end
+        end
+
+        if scheme_map.has_key?(parsed_uri.scheme)
+          parsed_uri.scheme = scheme_map[parsed_uri.scheme]
+        end
+
+        if parsed_uri.host && !parsed_uri.scheme
+          parsed_uri.scheme = default_scheme
+        end if set_default_scheme?
+
+        if parsed_uri.host && !domain
+          @domain = UrlParser::Domain.new(hostname)
+        end
+      end
+    end
+
+    def parse!
+      @uri = parse
+    end
+
+    def unembed
+      original = parse
+
+      candidates = original.query_values.select do |key, value|
+        embedded_params.include?(key) &&
+        value =~ Addressable::URI::URIREGEX
+      end.values if original.query_values
+
+      embed = candidates.find do |candidate|
+        parsed = Addressable::URI.parse(candidate)
+        %w(http https).include?(parsed.scheme) && parsed.host
+      end if candidates
+
+      embed ? self.class.call(embed, raw: raw?) : original
+    end
+    alias_method :embedded, :unembed
+
+    def unembed!
+      @uri = unembed
+    end
+    alias_method :embedded!, :unembed!
+
+    def normalize
+      parse.tap do |uri|
+        uri.path      = uri.path.squeeze('/')
+        uri.path      = uri.path.chomp('/') if uri.path.size != 1
+        uri.query     = nil if uri.query && uri.query.empty?
+        uri.query     = uri.query.strip if uri.query
+        uri.fragment  = nil
+
+        uri.normalize!
+      end
+    end
+
+    def normalize!
+      @uri = normalize
+    end
+
+    def canonicalize
+      parse.tap do |uri|
+        matches_global_param = proc do |key, value|
+          UrlParser::DB[:global].include?(key)
+        end
+
+        matches_host_based_param = proc do |key, value|
+          UrlParser::DB[:hosts].find do |host, param|
+            uri.host =~ Regexp.new(Regexp.escape(host)) && param.include?(key)
+          end
+        end
+
+        uri.query_values = uri.query_values(Array).tap do |params|
+          params.delete_if &matches_global_param
+          params.delete_if &matches_host_based_param
+        end if uri.query_values
+      end
+    end
+    alias_method :c14n, :canonicalize
+
+    def canonicalize!
+      @uri = canonicalize
+    end
+    alias_method :c14n!, :canonicalize!
+
+    def raw
+      uri.to_s
+    end
+
+    def raw!
+      @uri = raw
+    end
+
+    def sha1
+      Digest::SHA1.hexdigest(raw)
+    end
+    alias_method :hash, :sha1
+
+    def clean!
+      unescape!
+      parse!
+      unembed!
+      canonicalize!
+      normalize!
+      raw! if raw?
+    end
+
+    def ==(uri)
+      opts  = options.merge(raw: false)
+      one   = self.dup.tap { |uri| uri.clean! }
+      two   = self.class.new(uri, opts).tap { |uri| uri.clean! }
+
+      one.sha1 == two.sha1
+    end
+
+  end
+end