monovm-whois-ruby 1.0.0

data/lib/monovm/whois/client.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require_relative "configuration"
+require_relative "domain_name"
+require_relative "result"
+require_relative "parser/record"
+require_relative "referral/follower"
+require_relative "errors"
+
+module MonoVM
+  module Whois
+    # Looks up one domain, end to end.
+    #
+    # The client owns the *sequence* — resolve the TLD, pick an endpoint, fetch,
+    # classify, parse, maybe follow a referral — and none of the policy. Which server
+    # serves a TLD is the registry's business, how to talk to it is the transport's,
+    # what the answer means is the analyzer's, and what the record says is the
+    # parser's. Every one of them arrives by injection, so a spec can replace any
+    # single step without touching the others.
+    #
+    #   client = MonoVM::Whois::Client.new
+    #   client.lookup("monovm.com").status  # => :registered
+    class Client
+      # Fields where the registrar's copy is the better one, used when a thin
+      # registry's record is merged with the registrar's. Everywhere else the registry
+      # wins, because it is authoritative for whether the name exists, what state it is
+      # in and when it expires — and registrars are frequently stale on exactly those.
+      # The registrant and the contact block are the reverse: a thin registry never has
+      # them at all.
+      REGISTRAR_WINS = %i[registrant contacts].freeze
+
+      attr_reader :config, :server_registry, :transport_factory, :analyzer, :parsers, :follower
+
+      # Every argument defaults from +config+; pass one to replace that collaborator.
+      def initialize(config: nil, server_registry: nil, transport_factory: nil,
+                     analyzer: nil, parsers: nil, follower: nil)
+        @config = config || Configuration.new
+        @server_registry = server_registry || @config.server_registry
+        @transport_factory = transport_factory || @config.transport_factory
+        @analyzer = analyzer || @config.analyzer
+        @parsers = parsers || @config.parsers
+        @follower = follower || default_follower
+      end
+
+      # @param domain [String, DomainName]
+      # @return [Result] never raises for a lookup failure; the failure is the result
+      def lookup(domain)
+        name = domain.is_a?(DomainName) ? domain : DomainName.parse(domain)
+
+        return Result.invalid(name, reason: "#{name} is not a valid domain name") unless name.valid?
+
+        if name.bare?
+          return Result.invalid(
+            name,
+            reason: "#{name} has no TLD; give one, or use Checker to try popular TLDs"
+          )
+        end
+
+        resolution = server_registry.resolve(name)
+        return Result.invalid(name, reason: "no WHOIS or RDAP server is known for #{name}") if resolution.nil?
+
+        perform(name, resolution)
+      end
+
+      # @return [Boolean] true only when the lookup positively established availability
+      def available?(domain)
+        lookup(domain).available?
+      end
+
+      # Why a lookup came out the way it did, including every rule consulted.
+      #
+      # @return [Hash]
+      def explain(domain)
+        result = lookup(domain)
+
+        {
+          domain: result.name,
+          status: result.status,
+          tld: result.tld,
+          reason: result.reason,
+          trace: result.verdict&.trace || [],
+          endpoint: result.response&.endpoint&.to_s,
+          record: result.record&.to_h
+        }
+      end
+
+      private
+
+      def default_follower
+        return nil unless config.follow_referrals
+
+        Referral::Follower.new(transport_factory: transport_factory, max_hops: config.max_referrals)
+      end
+
+      def perform(name, resolution)
+        response, failure = fetch_first(resolution)
+
+        return failure_result(name, resolution, failure) if response.nil?
+
+        verdict = analyzer.analyze(
+          response: response,
+          tld: resolution.tld,
+          definition: resolution.definition
+        )
+
+        record = parsers.parse(response)
+        record, response = enrich(record, response, resolution, verdict)
+
+        Result.new(
+          domain: name,
+          status: verdict.status,
+          sld: resolution.sld,
+          tld: resolution.tld,
+          verdict: verdict,
+          record: record.empty? ? nil : record,
+          response: response
+        )
+      end
+
+      # Try the definition's endpoints in preference order, keeping the first answer.
+      #
+      # Falling through to port 43 when RDAP is unreachable is what keeps coverage
+      # high: a registry that has an RDAP entry in the IANA bootstrap but whose
+      # endpoint is broken still gets answered.
+      #
+      # @return [Array(Response, nil), Array(nil, Exception)]
+      def fetch_first(resolution)
+        endpoints = resolution.definition.endpoints(prefer: config.prefer)
+        return [nil, UnsupportedTldError.new(resolution.tld)] if endpoints.empty?
+
+        first_failure = nil
+
+        endpoints.each do |endpoint|
+          return [fetch(endpoint, resolution), nil]
+        rescue Error => e
+          # Remember the first failure: it came from the preferred endpoint and is
+          # the more relevant one to report.
+          first_failure ||= e
+        end
+
+        [nil, first_failure]
+      end
+
+      def fetch(endpoint, resolution)
+        transport_factory.for(endpoint).fetch(
+          query: query_for(endpoint, resolution),
+          endpoint: endpoint
+        )
+      end
+
+      # Punycode by default; the Unicode form for the few registries that insist on
+      # it, and only over port 43 since a URL must be ASCII either way.
+      def query_for(endpoint, resolution)
+        if endpoint.socket? && config.unicode_query?(resolution.tld)
+          resolution.registrable.to_s
+        else
+          resolution.query
+        end
+      end
+
+      # Follow a thin registry's referral to fill in the record, keeping the
+      # registry's verdict either way.
+      def enrich(record, response, resolution, verdict)
+        return [record, response] if follower.nil?
+        return [record, response] unless verdict.registered?
+
+        referral = follower.follow(response: response, record: record, query: resolution.query)
+        return [record, response] if referral.nil?
+
+        enriched = parsers.parse(referral)
+        return [record, response] if enriched.empty?
+
+        [merge_records(record, enriched), referral]
+      end
+
+      # Combine the registry's record with the registrar's, one policy stated once
+      # rather than a conditional per field.
+      def merge_records(registry_record, registrar_record)
+        registry = registry_record.attributes
+        registrar = registrar_record.attributes
+
+        merged = registrar.merge(registry) do |key, from_registrar, from_registry|
+          preferred, fallback =
+            if REGISTRAR_WINS.include?(key)
+              [from_registrar, from_registry]
+            else
+              [from_registry, from_registrar]
+            end
+
+          blank_value?(preferred) ? fallback : preferred
+        end
+
+        # Three fields are not a choice between two values.
+        merged[:registrar_whois_server] = registry[:registrar_whois_server]
+        merged[:fields] = registrar[:fields].merge(registry[:fields])
+        merged[:source] = [registry_record.source, registrar_record.source].compact.uniq.join("+")
+
+        Parser::Record.new(**merged)
+      end
+
+      def blank_value?(value)
+        value.nil? || (value.respond_to?(:empty?) && value.empty?)
+      end
+
+      # Map a transport failure onto a result. The distinction being preserved here:
+      # "we could not find out" is not "it is free".
+      def failure_result(name, resolution, failure)
+        case failure
+        when UnsupportedTldError
+          Result.invalid(
+            name,
+            reason: failure.message,
+            sld: resolution.sld,
+            tld: resolution.tld
+          )
+        else
+          Result.unknown(
+            name,
+            reason: failure&.message || "the lookup produced no response",
+            sld: resolution.sld,
+            tld: resolution.tld
+          )
+        end
+      end
+    end
+  end
+end

data/lib/monovm/whois/configuration.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require_relative "registry/server_registry"
+require_relative "transport/factory"
+require_relative "transport/middleware/cache"
+require_relative "transport/middleware/throttle"
+require_relative "transport/middleware/retry"
+require_relative "transport/middleware/instrumentation"
+require_relative "availability/analyzer"
+require_relative "availability/rule_set"
+require_relative "parser/selector"
+require_relative "paths"
+
+module MonoVM
+  module Whois
+    # Every knob in one place, plus the defaults for the objects {Client} depends on.
+    #
+    # {Client} takes its collaborators by injection, which is what makes it testable
+    # — but nobody wants to assemble a registry, a transport factory, an analyzer and
+    # a parser selector just to look up one domain. This class is the seam between
+    # those two needs: it holds the settings and builds sensible collaborators from
+    # them, while leaving every one of them replaceable.
+    #
+    #   MonoVM::Whois.configure do |config|
+    #     config.prefer = :whois
+    #     config.throttle_interval = 1.0
+    #     config.rules.insert_before("registry_marker", MyRule.new)
+    #   end
+    class Configuration
+      # TLDs whose registry wants an internationalised name in Unicode rather than
+      # Punycode, over port 43.
+      #
+      # DENIC answers +Status: invalid+ for +xn--mnchen-3ya.de+ but resolves
+      # +münchen.de+ correctly. Almost every other registry is the reverse — Verisign
+      # answers "No match" to a UTF-8 query, which reads as availability — so
+      # Punycode is the default and this is the short exception list. URLs are always
+      # ASCII, so it never applies to RDAP.
+      DEFAULT_UNICODE_QUERY_TLDS = [".de"].freeze
+
+      # Tried in order when a name arrives without a TLD.
+      DEFAULT_POPULAR_TLDS = [".com", ".net", ".org", ".info"].freeze
+
+      # Transport
+      attr_accessor :socket_connect_timeout, :socket_read_timeout,
+                    :http_open_timeout, :http_read_timeout, :verify_ssl, :user_agent
+
+      # Behaviour
+      attr_accessor :prefer, :follow_referrals, :max_referrals, :unicode_query_tlds,
+                    :popular_tlds, :concurrency, :override_path
+
+      # Middleware
+      attr_accessor :cache, :cache_ttl, :cache_max_entries,
+                    :throttle_interval, :retry_attempts, :retry_backoff, :instrumentation
+
+      # Injected collaborators; each defaults lazily and can be replaced outright.
+      attr_writer :server_registry, :transport_factory, :analyzer, :rules, :parsers
+
+      def initialize
+        @socket_connect_timeout = Transport::WhoisSocket::DEFAULT_CONNECT_TIMEOUT
+        @socket_read_timeout = Transport::WhoisSocket::DEFAULT_READ_TIMEOUT
+        @http_open_timeout = Transport::RdapHttp::DEFAULT_OPEN_TIMEOUT
+        @http_read_timeout = Transport::RdapHttp::DEFAULT_READ_TIMEOUT
+        @verify_ssl = true
+        @user_agent = nil
+
+        # RDAP first: a structured answer beats pattern-matching registry prose.
+        @prefer = :rdap
+        @follow_referrals = true
+        @max_referrals = 1
+        @unicode_query_tlds = DEFAULT_UNICODE_QUERY_TLDS.dup
+        @popular_tlds = DEFAULT_POPULAR_TLDS.dup
+        @concurrency = 8
+        @override_path = Paths.override
+
+        @cache = true
+        @cache_ttl = Transport::Middleware::Cache::DEFAULT_TTL
+        @cache_max_entries = Transport::Middleware::Cache::DEFAULT_MAX_ENTRIES
+        @throttle_interval = Transport::Middleware::Throttle::DEFAULT_INTERVAL
+        @retry_attempts = Transport::Middleware::Retry::DEFAULT_ATTEMPTS
+        @retry_backoff = Transport::Middleware::Retry::DEFAULT_BACKOFF
+        @instrumentation = nil
+      end
+
+      def server_registry
+        @server_registry ||= Registry::ServerRegistry.default(override_path: override_path)
+      end
+
+      def rules
+        @rules ||= Availability::RuleSet.default
+      end
+
+      def analyzer
+        @analyzer ||= Availability::Analyzer.new(rules: rules)
+      end
+
+      def parsers
+        @parsers ||= Parser::Selector.default
+      end
+
+      def transport_factory
+        @transport_factory ||= Transport::Factory.new(
+          transports: default_transports,
+          middleware: default_middleware
+        )
+      end
+
+      # True when +tld+'s registry wants the Unicode form over port 43.
+      def unicode_query?(tld)
+        unicode_query_tlds.include?(tld)
+      end
+
+      # A copy, so a per-call override cannot mutate the global default.
+      def dup
+        copy = super
+        copy.unicode_query_tlds = unicode_query_tlds.dup
+        copy.popular_tlds = popular_tlds.dup
+        copy
+      end
+
+      private
+
+      def default_transports
+        [
+          Transport::WhoisSocket.new(
+            connect_timeout: socket_connect_timeout,
+            read_timeout: socket_read_timeout
+          ),
+          Transport::RdapHttp.new(
+            open_timeout: http_open_timeout,
+            read_timeout: http_read_timeout,
+            verify_ssl: verify_ssl,
+            user_agent: user_agent
+          )
+        ]
+      end
+
+      # Order matters, and it reads outermost first. Instrumentation sees everything
+      # including cache hits; the cache answers before a throttle delay is paid;
+      # retries happen closest to the wire so a retried attempt is still throttled.
+      def default_middleware
+        layers = []
+        if instrumentation
+          layers << Transport::Middleware::Instrumentation.builder(subscriber: instrumentation)
+        end
+        if cache
+          layers << Transport::Middleware::Cache.builder(ttl: cache_ttl,
+                                                         max_entries: cache_max_entries)
+        end
+        if throttle_interval.to_f.positive?
+          layers << Transport::Middleware::Throttle.builder(interval: throttle_interval)
+        end
+        if retry_attempts.to_i > 1
+          layers << Transport::Middleware::Retry.builder(attempts: retry_attempts,
+                                                         backoff: retry_backoff)
+        end
+        layers
+      end
+    end
+  end
+end

data/lib/monovm/whois/domain_name.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+require_relative "punycode"
+
+module MonoVM
+  module Whois
+    # An immutable, normalised domain name.
+    #
+    # This class knows about the *name* and nothing else: how to clean up what a
+    # user pasted, how to move between Unicode and Punycode, and whether the
+    # result is a legal host name. Deciding which part of it is the TLD needs the
+    # server definitions, so that belongs to {Registry::ServerRegistry} — a name
+    # cannot know on its own whether +.co.uk+ is a suffix or two labels.
+    #
+    #   name = DomainName.parse("  HTTPS://WWW.MÜNCHEN.de:8080/path?q=1  ")
+    #   name.to_s   # => "www.münchen.de"
+    #   name.ascii  # => "www.xn--mnchen-3ya.de"
+    #   name.valid? # => true
+    class DomainName
+      ACE_PREFIX = "xn--"
+      MAX_LENGTH = 253
+      MAX_LABEL_LENGTH = 63
+
+      # A single legal DNS label: alphanumeric ends, hyphens allowed inside.
+      LABEL = /\A[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\z/
+
+      attr_reader :input, :unicode
+
+      class << self
+        # Normalise +input+ into a name. Never raises: an unusable string yields
+        # an instance whose {#valid?} is false, which is what bulk callers want so
+        # one bad entry does not abort a batch.
+        #
+        # @param input [String]
+        # @return [DomainName]
+        def parse(input)
+          raise InvalidDomainError, input unless input.is_a?(String)
+
+          new(input)
+        end
+
+        # Like {.parse} but raises when the result is not a usable host name.
+        #
+        # @raise [InvalidDomainError]
+        def parse!(input)
+          parse(input).tap do |name|
+            raise InvalidDomainError, input unless name.valid?
+          end
+        end
+      end
+
+      # Everything is computed here rather than memoised on demand, because the
+      # instance is frozen: a value object that lazily fills in an ivar would raise
+      # FrozenError the first time anyone asked for it.
+      def initialize(input)
+        @input = input
+        @unicode = normalise(input).freeze
+        @labels = @unicode.split(".", -1).freeze
+        @ascii = @labels.map { |label| label_to_ascii(label) }.join(".").freeze
+        @ascii_labels = @ascii.split(".", -1).freeze
+        @valid = compute_valid?
+        freeze
+      end
+
+      # The Unicode form — what a human reads.
+      def to_s
+        unicode
+      end
+
+      # The ASCII (Punycode) form — what goes on the wire and into URLs.
+      #
+      # Registries are queried in this form by default: Verisign answers "No match"
+      # to a UTF-8 query, which would read as availability.
+      attr_reader :ascii, :labels, :ascii_labels
+
+      # True when every label is a legal DNS label and the whole name fits in a
+      # DNS query. A single-label name like "monovm" is valid — callers append a
+      # TLD to it — so this deliberately does not require a dot.
+      def valid?
+        @valid
+      end
+
+      # True when the name needed Punycode, i.e. it has non-ASCII labels.
+      def idn?
+        ascii != unicode
+      end
+
+      # True when the name carries no dot, so no TLD was supplied.
+      def bare?
+        !unicode.include?(".")
+      end
+
+      # Build a new name by appending +suffix+ to this one.
+      #
+      #   DomainName.parse("monovm").join(".com").to_s # => "monovm.com"
+      def join(suffix)
+        suffix = suffix.to_s
+        suffix = ".#{suffix}" unless suffix.start_with?(".")
+        self.class.parse("#{unicode}#{suffix}")
+      end
+
+      def ==(other)
+        other.is_a?(DomainName) && other.unicode == unicode
+      end
+      alias eql? ==
+
+      def hash
+        [self.class, unicode].hash
+      end
+
+      def inspect
+        "#<#{self.class.name} #{unicode.inspect}#{" invalid" unless valid?}>"
+      end
+
+      private
+
+      # Reduce what people actually paste — URLs, mixed case, credentials, a
+      # trailing root dot — down to a bare host name.
+      def normalise(raw)
+        text = raw.to_s.strip
+        text = text.split("://", 2).last.to_s if text.include?("://")
+        text = text.rpartition("@").last unless text.rpartition("@").last.empty?
+
+        ["/", "?", "#"].each { |separator| text = text.split(separator, 2).first.to_s }
+
+        # A bare colon is always a port here; an IPv6 literal is not a domain.
+        text = text.split(":", 2).first.to_s
+
+        text = text.strip.gsub(/\A\.+|\.+\z/, "")
+        downcase(text)
+      end
+
+      # Unicode-aware downcasing plus NFC, so "MÜNCHEN" and a decomposed "münchen"
+      # both reach Punycode as the same sequence of code points.
+      def downcase(text)
+        return text.downcase if text.ascii_only?
+
+        text.unicode_normalize(:nfc).downcase
+      rescue ArgumentError, Encoding::CompatibilityError
+        # Invalid byte sequences cannot be normalised; leave them for #valid? to reject.
+        text
+      end
+
+      def label_to_ascii(label)
+        return label if label.ascii_only?
+
+        "#{ACE_PREFIX}#{Punycode.encode(label)}"
+      rescue Punycode::Error
+        # Not convertible; hand it back unchanged so #valid? rejects the name
+        # rather than this raising from a plain reader method.
+        label
+      end
+
+      def compute_valid?
+        return false if unicode.empty?
+
+        candidate = ascii
+        return false if candidate.empty? || candidate.length > MAX_LENGTH
+        return false unless candidate.ascii_only?
+
+        ascii_labels.all? do |label|
+          !label.empty? && label.length <= MAX_LABEL_LENGTH && LABEL.match?(label)
+        end
+      end
+    end
+  end
+end

data/lib/monovm/whois/endpoint.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "uri"
+require_relative "errors"
+
+module MonoVM
+  module Whois
+    # Where a query gets sent, and over which protocol.
+    #
+    # Server definitions describe two very different things with one string:
+    # +socket://whois.nic.uk+ is a host to open a TCP connection to, while
+    # +https://rdap.verisign.com/com/v1/domain/+ is a URL prefix. This object
+    # normalises both so {Transport::Factory} can pick a transport by asking
+    # {#kind} rather than by re-parsing strings all over the codebase.
+    class Endpoint
+      SOCKET_SCHEME = "socket"
+      DEFAULT_WHOIS_PORT = 43
+
+      # Substituted with the queried name when present, so an unusual endpoint can
+      # put the domain somewhere other than the end of the URL.
+      PLACEHOLDER = "{domain}"
+
+      attr_reader :uri, :scheme, :host, :port
+
+      class << self
+        # @param uri [String] e.g. +socket://whois.nic.uk+, +socket://host:4343+,
+        #   or +https://rdap.example/domain/+
+        # @return [Endpoint]
+        # @raise [DefinitionsError] when the string is not a usable endpoint
+        def parse(uri)
+          raise DefinitionsError, "endpoint is empty" if uri.nil? || uri.to_s.strip.empty?
+
+          new(uri.to_s.strip)
+        end
+      end
+
+      def initialize(uri)
+        @uri = uri
+        @scheme = uri.split("://", 2).first.to_s.downcase
+        parse_target!
+        freeze
+      end
+
+      # True for port 43 WHOIS.
+      def socket?
+        scheme == SOCKET_SCHEME
+      end
+
+      # True for RDAP or any other HTTP-delivered service.
+      def http?
+        %w[http https].include?(scheme)
+      end
+
+      def tls?
+        scheme == "https"
+      end
+
+      # A coarse label used to select a transport and to tag a {Response}.
+      def kind
+        socket? ? :whois : :rdap
+      end
+
+      # The URL to fetch for +name+. Only meaningful for HTTP endpoints.
+      #
+      #   Endpoint.parse("https://rdap.example/domain/").url_for("a.com")
+      #   # => "https://rdap.example/domain/a.com"
+      def url_for(name)
+        raise DefinitionsError, "#{uri} is not an HTTP endpoint" unless http?
+        return uri.sub(PLACEHOLDER, name) if uri.include?(PLACEHOLDER)
+
+        "#{uri}#{name}"
+      end
+
+      def to_s
+        uri
+      end
+
+      def ==(other)
+        other.is_a?(Endpoint) && other.uri == uri
+      end
+      alias eql? ==
+
+      def hash
+        [self.class, uri].hash
+      end
+
+      def inspect
+        "#<#{self.class.name} #{uri}>"
+      end
+
+      private
+
+      def parse_target!
+        if socket?
+          parse_socket_target!
+        elsif http?
+          parse_http_target!
+        else
+          raise DefinitionsError, "unsupported endpoint scheme in #{uri.inspect}"
+        end
+      end
+
+      # +socket://host+ or +socket://host:port+. A trailing slash is tolerated
+      # because a few definition files carry one.
+      def parse_socket_target!
+        target = uri.split("://", 2).last.to_s.delete_suffix("/")
+        host, _, port = target.rpartition(":")
+
+        if host.empty? || !port.match?(/\A\d+\z/)
+          @host = target
+          @port = DEFAULT_WHOIS_PORT
+        else
+          @host = host
+          @port = port.to_i
+        end
+
+        raise DefinitionsError, "no host in endpoint #{uri.inspect}" if @host.empty?
+      end
+
+      def parse_http_target!
+        parsed = URI.parse(uri.sub(PLACEHOLDER, "placeholder"))
+        raise DefinitionsError, "no host in endpoint #{uri.inspect}" if parsed.host.nil?
+
+        @host = parsed.host
+        @port = parsed.port
+      rescue URI::InvalidURIError => e
+        raise DefinitionsError, "invalid endpoint #{uri.inspect}: #{e.message}"
+      end
+    end
+  end
+end