domain_extractor 0.2.7 → 0.2.9

data/lib/domain_extractor/parsed_url.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+require_relative 'auth'
+require_relative 'query_params'
+require_relative 'uri_helpers'
+
 module DomainExtractor
   # ParsedURL wraps the parsing result and provides convenient accessor methods
   # with support for bang (!) and question mark (?) variants.
@@ -15,16 +19,26 @@ module DomainExtractor
   #   parsed.host           # => nil
   #   parsed.host?          # => false
   #   parsed.host!          # raises InvalidURLError
+  # rubocop:disable Metrics/ClassLength
   class ParsedURL
+    EMPTY_STRING = ''
+
     # Expose the underlying hash for backward compatibility
     attr_reader :result
 
+    # Store the original URI object for advanced operations
+    attr_reader :uri
+
     # List of valid result keys that should have method accessors
-    RESULT_KEYS = %i[subdomain domain tld root_domain host path query_params].freeze
+    RESULT_KEYS = %i[
+      subdomain domain tld root_domain host path query_params
+      scheme port fragment user password userinfo decoded_user decoded_password
+    ].freeze
 
-    def initialize(result)
-      @result = result || {}
-      freeze
+    def initialize(result, uri = nil)
+      @result = (result || {}).dup
+      @uri = uri
+      sync_uri_state!
     end
 
     # Hash-style access for backward compatibility
@@ -88,7 +102,9 @@ module DomainExtractor
     end
 
     def to_s
-      @result.to_s
+      return EMPTY_STRING unless valid? && @uri
+
+      @uri.to_s
     end
 
     # Allow to_h conversion for hash compatibility
@@ -99,8 +115,222 @@ module DomainExtractor
     # Allow to_hash as well for better Ruby compatibility
     alias to_hash to_h
 
+    # Alias for URI compatibility
+    alias to_str to_s
+
+    def scheme
+      @result[:scheme]
+    end
+
+    def host
+      @result[:host]
+    end
+
+    def port
+      @result[:port]
+    end
+
+    def path
+      @result[:path]
+    end
+
+    def fragment
+      @result[:fragment]
+    end
+
+    def user
+      @result[:user]
+    end
+
+    def password
+      @result[:password]
+    end
+
+    def userinfo
+      @result[:userinfo]
+    end
+
+    # hostname returns host without IPv6 brackets (URI compatibility)
+    def hostname
+      return nil unless @uri || host
+
+      @uri&.hostname || host.to_s.gsub(/^\[|\]$/, '')
+    end
+
+    # query returns the query string (not parsed params)
+    def query
+      return nil unless @uri
+
+      @uri.query
+    end
+
+    # Setter methods for URI compatibility
+    def scheme=(value)
+      mutate_uri! { @uri.scheme = normalize_scheme(value) }
+    end
+
+    def host=(value)
+      mutate_uri! { replace_host(value) }
+    end
+
+    def hostname=(value)
+      self.host = value
+    end
+
+    def port=(value)
+      mutate_uri! { @uri.port = value }
+    end
+
+    def path=(value)
+      mutate_uri! { @uri.path = value.to_s }
+    end
+
+    def query=(value)
+      mutate_uri! { @uri.query = value }
+    end
+
+    def fragment=(value)
+      mutate_uri! { @uri.fragment = value }
+    end
+
+    def user=(value)
+      mutate_uri! { @uri.user = value }
+    end
+
+    def password=(value)
+      mutate_uri! { @uri.password = value }
+    end
+
+    def userinfo=(value)
+      mutate_uri! { @uri.userinfo = value }
+    end
+
+    # Advanced URI methods
+
+    # Generate Basic Authentication header from current credentials
+    # @return [String, nil] Authorization header value or nil if no credentials
+    def basic_auth_header
+      return nil if user.nil? || password.nil?
+
+      URIHelpers.basic_auth_header(decoded_user || user, decoded_password || password)
+    end
+
+    # Generate Bearer token header
+    # @param token [String] The bearer token
+    # @return [String] Authorization header value
+    def bearer_auth_header(token)
+      URIHelpers.bearer_auth_header(token)
+    end
+
+    # Find proxy for this URL
+    # @return [URI::Generic, nil] Proxy URI or nil
+    def find_proxy
+      return nil unless @uri
+
+      URIHelpers.find_proxy(@uri)
+    end
+
+    # Merge with a relative URI
+    # @param relative [String, URI::Generic] The relative URI
+    # @return [ParsedURL] New ParsedURL with merged URI
+    def merge(relative)
+      return self unless @uri
+
+      merged_uri = URIHelpers.merge_uri(@uri, relative)
+      DomainExtractor.parse(merged_uri.to_s)
+    end
+
+    # Normalize the URI (lowercase scheme/host, remove default ports)
+    # @return [ParsedURL] New ParsedURL with normalized URI
+    def normalize
+      return self unless @uri
+
+      normalized_uri = URIHelpers.normalize_uri(@uri)
+      DomainExtractor.parse(normalized_uri.to_s)
+    end
+
+    # Check if this is an absolute URI
+    # @return [Boolean] True if absolute
+    def absolute?
+      !@result[:scheme].nil?
+    end
+
+    # Check if this is a relative URI
+    # @return [Boolean] True if relative
+    def relative?
+      @result[:scheme].nil?
+    end
+
+    # Get the default port for the scheme
+    # @return [Integer, nil] Default port or nil
+    def default_port
+      URIHelpers.default_port_for(@uri || scheme)
+    end
+
+    # Build a complete URL string from components
+    # @return [String] The complete URL
+    def build_url
+      to_s
+    end
+
     private
 
+    def mutate_uri!
+      return unless @uri
+
+      yield
+      sync_from_uri!
+    end
+
+    def sync_uri_state!
+      return unless @uri && valid?
+
+      current_userinfo = @uri.userinfo
+      @uri.scheme = normalize_scheme(@result[:scheme]) if @result[:scheme]
+      if @result[:host]
+        @uri.host = normalize_host(@result[:host])
+        @uri.userinfo = current_userinfo if current_userinfo
+      end
+      sync_from_uri!
+    end
+
+    def sync_from_uri!
+      attributes = DomainExtractor::Parser.host_attributes(@uri.host)
+
+      unless attributes
+        @result.clear
+        return
+      end
+
+      @result.replace(
+        attributes.merge(
+          path: @uri.path || EMPTY_STRING,
+          query_params: QueryParams.call(@uri.query),
+          scheme: normalize_scheme(@uri.scheme),
+          port: @uri.port,
+          fragment: @uri.fragment
+        ).merge(Auth.extract(@uri))
+      )
+    end
+
+    def normalize_host(value)
+      return nil if value.nil?
+
+      value.to_s.downcase
+    end
+
+    def normalize_scheme(value)
+      return nil if value.nil?
+
+      value.to_s.downcase
+    end
+
+    def replace_host(value)
+      current_userinfo = @uri.userinfo
+      @uri.host = normalize_host(value)
+      @uri.userinfo = current_userinfo if current_userinfo
+    end
+
     # Handle bang methods that raise errors for missing values
     def handle_bang_method(method_str)
       key = method_str[0...-1].to_sym
@@ -128,4 +358,5 @@ module DomainExtractor
       true
     end
   end
+  # rubocop:enable Metrics/ClassLength
 end

data/lib/domain_extractor/parser.rb

@@ -7,18 +7,29 @@ require_relative 'normalizer'
 require_relative 'result'
 require_relative 'validators'
 require_relative 'parsed_url'
+require_relative 'auth'
+
+# Register custom URI schemes for database and other protocols
+# This allows URI.parse to handle redis://, mysql://, postgresql://, etc.
+%w[redis rediss mysql postgresql mongodb sftp ftps].each do |scheme|
+  URI.scheme_list[scheme.upcase] = URI::Generic
+rescue StandardError
+  # Ignore if can't register
+end
 
 module DomainExtractor
   # Parser orchestrates the pipeline for url normalization, validation, and domain extraction.
   module Parser
+    SCHEME_PATTERN = %r{\A([a-z][a-z0-9+.-]*)://}i
+    RETRYABLE_URI_MESSAGES = ['bad URI', 'is not URI'].freeze
+
     module_function
 
     def call(raw_url)
-      components = extract_components(raw_url)
-      return ParsedURL.new(nil) unless components
+      uri, host_attributes = extract_components(raw_url)
+      return ParsedURL.new(nil) unless uri && host_attributes
 
-      uri, domain, host = components
-      build_result(domain: domain, host: host, uri: uri)
+      build_result(host_attributes: host_attributes, uri: uri)
     rescue ::URI::InvalidURIError, ::PublicSuffix::Error
       ParsedURL.new(nil)
     end
@@ -29,16 +40,29 @@ module DomainExtractor
       false
     end
 
-    def build_uri(raw_url)
+    def host_attributes(host)
+      return if invalid_host?(host)
+
+      normalized_host = host.downcase
+      domain = parse_domain(normalized_host)
+
+      return domain_attributes(domain, normalized_host) if domain
+
+      hostname_attributes(normalized_host) if Validators.valid_hostname?(normalized_host)
+    end
+
+    def build_uri(raw_url, retry_count = 0)
       normalized = Normalizer.call(raw_url)
       return unless normalized
 
       ::URI.parse(normalized)
+    rescue ::URI::InvalidURIError => e
+      retry_parse_with_registered_scheme(e, normalized, raw_url, retry_count)
     end
     private_class_method :build_uri
 
     def invalid_host?(host)
-      host.nil? || Validators.ip_address?(host) || !::PublicSuffix.valid?(host)
+      host.nil? || Validators.ip_address?(host)
     end
     private_class_method :invalid_host?
 
@@ -46,23 +70,76 @@ module DomainExtractor
       uri = build_uri(raw_url)
       return unless uri
 
-      host = uri.host&.downcase
-      return if invalid_host?(host)
+      attributes = host_attributes(uri.host)
+      return unless attributes
 
-      domain = ::PublicSuffix.parse(host)
-      [uri, domain, host]
+      [uri, attributes]
     end
     private_class_method :extract_components
 
-    def build_result(domain:, host:, uri:)
-      Result.build(
+    def parse_domain(host)
+      ::PublicSuffix.parse(host)
+    rescue ::PublicSuffix::Error
+      nil
+    end
+    private_class_method :parse_domain
+
+    def domain_attributes(domain, host)
+      {
         subdomain: domain.trd,
         root_domain: domain.domain,
         domain: domain.sld,
         tld: domain.tld,
-        host: host,
+        host: host
+      }
+    end
+    private_class_method :domain_attributes
+
+    def hostname_attributes(host)
+      {
+        subdomain: nil,
+        root_domain: host,
+        domain: host,
+        tld: nil,
+        host: host
+      }
+    end
+    private_class_method :hostname_attributes
+
+    def retry_parse_with_registered_scheme(error, normalized, raw_url, retry_count)
+      return nil unless retryable_scheme_registration?(error.message, normalized, retry_count)
+
+      register_scheme(normalized[SCHEME_PATTERN, 1])
+      build_uri(raw_url, 1)
+    rescue StandardError
+      nil
+    end
+    private_class_method :retry_parse_with_registered_scheme
+
+    def retryable_scheme_registration?(message, normalized, retry_count)
+      retry_count.zero? &&
+        RETRYABLE_URI_MESSAGES.any? { |fragment| message.include?(fragment) } &&
+        normalized.match?(SCHEME_PATTERN)
+    end
+    private_class_method :retryable_scheme_registration?
+
+    def register_scheme(scheme)
+      URI.scheme_list[scheme.upcase] = URI::Generic
+    end
+    private_class_method :register_scheme
+
+    def build_result(host_attributes:, uri:)
+      auth_components = Auth.extract(uri)
+
+      Result.build(
+        **host_attributes,
         path: uri.path,
-        query: uri.query
+        query: uri.query,
+        scheme: uri.scheme,
+        port: uri.port,
+        fragment: uri.fragment,
+        **auth_components,
+        uri: uri
       )
     end
     private_class_method :build_result

data/lib/domain_extractor/result.rb

@@ -12,22 +12,53 @@ module DomainExtractor
     module_function
 
     def build(**attributes)
-      hash = {
+      ParsedURL.new(result_hash(attributes), attributes[:uri])
+    end
+
+    def normalize_subdomain(value)
+      value.nil? || value.empty? ? nil : value
+    end
+    private_class_method :normalize_subdomain
+
+    def result_hash(attributes)
+      domain_attributes(attributes)
+        .merge(uri_attributes(attributes))
+        .merge(auth_attributes(attributes))
+        .freeze
+    end
+    private_class_method :result_hash
+
+    def domain_attributes(attributes)
+      {
         subdomain: normalize_subdomain(attributes[:subdomain]),
         root_domain: attributes[:root_domain],
         domain: attributes[:domain],
         tld: attributes[:tld],
-        host: attributes[:host],
-        path: attributes[:path] || EMPTY_PATH,
-        query_params: QueryParams.call(attributes[:query])
-      }.freeze
+        host: attributes[:host]
+      }
+    end
+    private_class_method :domain_attributes
 
-      ParsedURL.new(hash)
+    def uri_attributes(attributes)
+      {
+        path: attributes[:path] || EMPTY_PATH,
+        query_params: QueryParams.call(attributes[:query]),
+        scheme: attributes[:scheme],
+        port: attributes[:port],
+        fragment: attributes[:fragment]
+      }
     end
+    private_class_method :uri_attributes
 
-    def normalize_subdomain(value)
-      value.nil? || value.empty? ? nil : value
+    def auth_attributes(attributes)
+      {
+        user: attributes[:user],
+        password: attributes[:password],
+        userinfo: attributes[:userinfo],
+        decoded_user: attributes[:decoded_user],
+        decoded_password: attributes[:decoded_password]
+      }
     end
-    private_class_method :normalize_subdomain
+    private_class_method :auth_attributes
   end
 end

data/lib/domain_extractor/uri_helpers.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'uri'
+
+module DomainExtractor
+  # URIHelpers provides advanced URI manipulation methods
+  # Including merge, normalize, authentication helpers, and proxy detection
+  # rubocop:disable Metrics/ModuleLength
+  module URIHelpers
+    CREDENTIAL_ESCAPE_PATTERN = /[^A-Za-z0-9\-._~]/
+    DEFAULT_PORTS = {
+      'ftp' => 21,
+      'ftps' => 990,
+      'http' => 80,
+      'https' => 443,
+      'mongodb' => 27_017,
+      'mysql' => 3306,
+      'postgresql' => 5432,
+      'redis' => 6379,
+      'rediss' => 6380,
+      'sftp' => 22,
+      'ssh' => 22
+    }.freeze
+    HTTP_PROXY_KEYS = %w[http_proxy HTTP_PROXY].freeze
+    ALL_PROXY_KEYS = %w[all_proxy ALL_PROXY].freeze
+
+    module_function
+
+    # Generate Basic Authentication header
+    # @param username [String] The username
+    # @param password [String] The password
+    # @return [String] The Authorization header value
+    def basic_auth_header(username, password)
+      credentials = "#{username}:#{password}"
+      encoded = Base64.strict_encode64(credentials)
+      "Basic #{encoded}"
+    end
+
+    # Generate Bearer token header
+    # @param token [String] The bearer token
+    # @return [String] The Authorization header value
+    def bearer_auth_header(token)
+      "Bearer #{token}"
+    end
+
+    # Encode credentials for URL (percent-encoding)
+    # @param value [String] The value to encode
+    # @return [String] Percent-encoded value
+    def encode_credential(value)
+      URI::DEFAULT_PARSER.escape(value.to_s, CREDENTIAL_ESCAPE_PATTERN)
+    end
+
+    # Decode percent-encoded credential
+    # @param value [String] The encoded value
+    # @return [String] Decoded value
+    def decode_credential(value)
+      URI::DEFAULT_PARSER.unescape(value.to_s)
+    rescue StandardError
+      value
+    end
+
+    # Find proxy from environment variables
+    # Checks http_proxy, HTTP_PROXY, and no_proxy
+    # @param uri [URI::Generic, String] The URI to check
+    # @return [URI::Generic, nil] The proxy URI or nil
+    def find_proxy(uri)
+      uri_obj = coerce_uri(uri)
+      return nil unless uri_obj
+      return nil if should_bypass_proxy?(uri_obj)
+
+      proxy_url = proxy_url_for(uri_obj.scheme)
+      return nil unless proxy_url
+
+      URI.parse(proxy_url)
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    # Check if URI should bypass proxy based on no_proxy
+    # @param uri [URI::Generic] The URI to check
+    # @return [Boolean] True if should bypass proxy
+    def should_bypass_proxy?(uri)
+      no_proxy = ENV['no_proxy'] || ENV.fetch('NO_PROXY', nil)
+      return false unless no_proxy
+
+      host = proxy_host(uri)
+      return false unless host
+
+      no_proxy
+        .split(',')
+        .map(&:strip)
+        .reject(&:empty?)
+        .any? { |pattern| proxy_pattern_match?(host, pattern) }
+    end
+    private_class_method :should_bypass_proxy?
+
+    def coerce_uri(uri)
+      uri.is_a?(String) ? URI.parse(uri) : uri
+    end
+    private_class_method :coerce_uri
+
+    def proxy_url_for(scheme)
+      proxy_env_keys(scheme).each do |key|
+        value = env_value(key)
+        return value if value
+      end
+
+      nil
+    end
+    private_class_method :proxy_url_for
+
+    def env_value(key)
+      value = ENV.fetch(key, nil)
+      value unless value.nil? || value.empty?
+    end
+    private_class_method :env_value
+
+    def proxy_host(uri)
+      uri.hostname || uri.host
+    end
+    private_class_method :proxy_host
+
+    def proxy_pattern_match?(host, pattern)
+      return true if pattern == '*'
+
+      normalized_pattern = pattern.delete_prefix('.')
+      host == normalized_pattern || host.end_with?(".#{normalized_pattern}")
+    end
+    private_class_method :proxy_pattern_match?
+
+    def proxy_env_keys(scheme)
+      scheme_keys = if scheme && !scheme.empty?
+                      ["#{scheme.downcase}_proxy", "#{scheme.upcase}_PROXY"]
+                    else
+                      []
+                    end
+
+      (scheme_keys + HTTP_PROXY_KEYS + ALL_PROXY_KEYS).uniq
+    end
+    private_class_method :proxy_env_keys
+
+    # Normalize a URI (lowercase scheme and host, remove default ports)
+    # @param uri [URI::Generic] The URI to normalize
+    # @return [URI::Generic] Normalized URI
+    def normalize_uri(uri)
+      uri.normalize
+    end
+
+    # Merge a relative URI with a base URI
+    # @param base [URI::Generic] The base URI
+    # @param relative [String, URI::Generic] The relative URI
+    # @return [URI::Generic] The merged URI
+    def merge_uri(base, relative)
+      base.merge(relative)
+    end
+
+    def default_port_for(uri_or_scheme)
+      case uri_or_scheme
+      when URI::Generic
+        uri_or_scheme.default_port || DEFAULT_PORTS[uri_or_scheme.scheme]
+      else
+        DEFAULT_PORTS[uri_or_scheme.to_s]
+      end
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end

data/lib/domain_extractor/validators.rb

@@ -8,6 +8,11 @@ module DomainExtractor
     IPV4_REGEX = /\A#{IPV4_SEGMENT}(?:\.#{IPV4_SEGMENT}){3}\z/
     IPV6_REGEX = /\A\[?[0-9a-fA-F:]+\]?\z/
 
+    # Valid hostname pattern (RFC 1123)
+    # Allows: letters, numbers, hyphens, dots
+    # Must start with alphanumeric, can contain hyphens, must end with alphanumeric
+    HOSTNAME_REGEX = /\A[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*\z/i
+
     # Frozen string constants
     DOT = '.'
     COLON = ':'
@@ -27,5 +32,15 @@ module DomainExtractor
         false
       end
     end
+
+    # Check if a string is a valid hostname
+    # @param host [String] The hostname to validate
+    # @return [Boolean] True if valid hostname
+    def valid_hostname?(host)
+      return false if host.nil? || host.empty?
+      return false if host.length > 253 # Max hostname length
+
+      HOSTNAME_REGEX.match?(host)
+    end
   end
 end

data/lib/domain_extractor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DomainExtractor
-  VERSION = '0.2.7'
+  VERSION = '0.2.9'
 end