mt-uv-rays 2.4.7

data/lib/mt-uv-rays/http/request.rb

@@ -0,0 +1,262 @@
+# encoding: ASCII-8BIT
+# frozen_string_literal: true
+
+require 'uri'
+require 'cgi'
+require 'rubyntlm'
+require 'net/http/digest_auth'
+
+module MTUV
+    module Http
+        class Request < ::MTLibuv::Q::DeferredPromise
+            include Encoding
+
+
+            COOKIE = 'cookie'
+            CONNECTION = :Connection
+            CRLF="\r\n"
+
+
+            attr_reader :path, :method, :options
+
+
+            def cookies_hash
+                @cookiejar.get_hash(@uri)
+            end
+
+            def set_cookie(value)
+                @cookiejar.set(@uri, value)
+            end
+
+
+            def initialize(endpoint, options)
+                super(endpoint.thread, endpoint.thread.defer)
+
+                @host = endpoint.host
+                @port = endpoint.port
+                @http_proxy = endpoint.http_proxy? ? endpoint.proxy : nil
+                @encoded_host = endpoint.encoded_host
+
+                path = options[:path]
+                if path.is_a?(::URI)
+                    @path = path.to_s.split(@encoded_host)[1] || '/'
+                elsif path.include?("://")
+                    @path = path.split(@encoded_host)[1] || '/'
+                else
+                    @path = path
+                end
+
+                @method = options[:method]
+                @cookiejar = endpoint.cookiejar
+                @middleware = endpoint.middleware
+                @uri = "#{endpoint.scheme}://#{@encoded_host}#{@path}"
+                @path = @uri if @http_proxy
+                endpoint = nil
+
+                @options = options
+                @ntlm_creds = options[:ntlm]
+                @digest_creds = options[:digest]
+                @challenge_retries = 0
+
+                # Don't hold references to vars we don't require anymore
+                self.finally {
+                    @host = @port = nil
+                    @cookiejar = nil
+                    @middleware = nil
+                }
+                @error = proc { |reason| reject(reason) }
+            end
+
+
+
+            def resolve(response, parser = nil)
+                if response.status == 401 && @challenge_retries == 0 && response[:"WWW-Authenticate"]
+                    challenge = Array(response[:"WWW-Authenticate"]).reject { |auth| auth.downcase == 'negotiate' }[0]
+
+                    begin
+                        if @ntlm_creds && challenge[0..3] == 'NTLM'
+                            @options[:headers] ||= {}
+                            @options[:headers][:Authorization] = ntlm_auth_header(challenge)
+                            @challenge_retries += 1
+
+                            execute(@transport)
+                            return false
+                        elsif @digest_creds && challenge[0..5] == 'Digest'
+                            @options[:headers] ||= {}
+                            @options[:headers][:Authorization] = digest_auth_header(challenge)
+                            @challenge_retries += 1
+
+                            execute(@transport)
+                            return false
+                        end
+                    rescue => e
+                        reject e
+                        true
+                    end
+                end
+
+                @transport = nil
+                @defer.resolve(response)
+                true
+            end
+
+            def reject(reason)
+                @defer.reject(reason)
+            end
+
+            def execute(transport)
+                # configure ntlm request headers
+                if @options[:ntlm]
+                    @options[:headers] ||= {}
+                    @options[:headers][:Authorization] ||= ntlm_auth_header
+                end
+
+                head, body = build_request, @options[:body]
+                @transport = transport
+
+                @middleware.each do |m|
+                    begin
+                        head, body = m.request(self, head, body) if m.respond_to?(:request)
+                    rescue => e
+                        reject e
+                        return
+                    end
+                end
+
+                body = body.is_a?(Hash) ? form_encode_body(body) : body
+                file = @options[:file]
+                query = @options[:query]
+
+                # Set the Content-Length if file is given
+                head['content-length'] = File.size(file) if file
+
+                # Set the Content-Length if body is given,
+                # or we're doing an empty post or put
+                if body
+                    head['content-length'] = body.bytesize
+                elsif method == :post or method == :put
+                    # wont happen if body is set and we already set content-length above
+                    head['content-length'] ||= 0
+                end
+
+                # Set content-type header if missing and body is a Ruby hash
+                if !head['content-type'] and @options[:body].is_a? Hash
+                    head['content-type'] = 'application/x-www-form-urlencoded'
+                end
+
+                request_header = encode_request(method, @path, query)
+                if @http_proxy && (@http_proxy[:username] || @http_proxy[:password])
+                    request_header << encode_auth('Proxy-Authorization', [@http_proxy[:username], @http_proxy[:password]])
+                end
+                request_header << encode_headers(head)
+                request_header << CRLF
+
+                if body
+                    request_header << body
+                    transport.write(request_header).catch &@error
+                elsif file
+                    transport.write(request_header).catch &@error
+
+                    # Send file
+                    fileRef = @reactor.file file, File::RDONLY do
+                        # File is open and available for reading
+                        pSend = fileRef.send_file(transport, using: :raw, wait: :promise)
+                        pSend.catch &@error
+                        pSend.finally do
+                            fileRef.close
+                        end
+                    end
+                    fileRef.catch &@error
+                else
+                    transport.write(request_header).catch &@error
+                end
+            end
+
+            def notify(*args)
+                @defer.notify(*args)
+            end
+
+            def set_headers(head)
+                @headers_callback.call(head) if @headers_callback
+            end
+
+            def on_headers(&callback)
+                @headers_callback = callback
+            end
+
+            def streaming?
+                @options[:streaming]
+            end
+
+
+            protected
+
+
+            def build_request
+                head = @options[:headers] ? munge_header_keys(@options[:headers]) : {}
+
+                # Set the cookie header if provided
+                @cookies = @cookiejar.get(@uri)
+                if cookie = head[COOKIE]
+                    @cookies << encode_cookie(cookie)
+                end
+                head[COOKIE] = @cookies.compact.uniq.join("; ").squeeze(";") unless @cookies.empty?
+
+                # Set connection close unless keep-alive
+                if !@options[:keepalive]
+                    head['connection'] = 'close'
+                end
+
+                # Set the Host header if it hasn't been specified already
+                head['host'] ||= @encoded_host
+
+                # Set the User-Agent if it hasn't been specified
+                if !head.key?('user-agent')
+                    head['user-agent'] = "MTUV HttpClient"
+                elsif head['user-agent'].nil?
+                    head.delete('user-agent')
+                end
+
+                head
+            end
+
+            def ntlm_auth_header(challenge = nil)
+                if @ntlm_auth && challenge.nil?
+                    return @ntlm_auth
+                elsif challenge
+                    scheme, param_str = parse_ntlm_challenge_header(challenge)
+                    if param_str.nil?
+                        @ntlm_auth = nil
+                        return ntlm_auth_header(@ntlm_creds)
+                    else
+                        t2 = Net::NTLM::Message.decode64(param_str)
+                        t3 = t2.response(@ntlm_creds, ntlmv2: true)
+                        @ntlm_auth = "NTLM #{t3.encode64}"
+                        return @ntlm_auth
+                    end
+                else
+                    domain = @ntlm_creds[:domain]
+                    t1 = Net::NTLM::Message::Type1.new()
+                    t1.domain = domain if domain
+                    @ntlm_auth = "NTLM #{t1.encode64}"
+                    return @ntlm_auth
+                end
+            end
+
+            def parse_ntlm_challenge_header(challenge)
+                scheme, param_str = challenge.scan(/\A(\S+)(?:\s+(.*))?\z/)[0]
+                return nil if scheme.nil?
+                return scheme, param_str
+            end
+
+            def digest_auth_header(challenge)
+                uri = URI.parse @uri
+                uri.userinfo = "#{CGI::escape(@digest_creds[:user])}:#{CGI::escape(@digest_creds[:password])}"
+
+                digest_auth = Net::HTTP::DigestAuth.new
+                auth = digest_auth.auth_header uri, challenge, method.to_s.upcase
+                auth
+            end
+        end
+    end
+end

data/lib/mt-uv-rays/http_endpoint.rb

@@ -0,0 +1,336 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'cookiejar'         # Manages cookies
+require 'http-parser'       # Parses HTTP request / responses
+require 'addressable/uri'   # URI parser
+require 'mt-uv-rays/http/encoding'
+require 'mt-uv-rays/http/request'
+require 'mt-uv-rays/http/parser'
+
+module MTUV
+    class CookieJar
+        def initialize
+            @jar = ::CookieJar::Jar.new
+        end
+
+        def set(uri, string)
+            @jar.set_cookie(uri, string) rescue nil # drop invalid cookies
+        end
+
+        def get(uri)
+            uri = URI.parse(uri) rescue nil
+            uri ? @jar.get_cookies(uri).map(&:to_s) : []
+        end
+
+        def get_hash(uri)
+            uri = URI.parse(uri) rescue nil
+            cookies = {}
+            if uri
+                @jar.get_cookies(uri).each do |cookie|
+                    cookies[cookie.name.to_sym] = cookie.value
+                end
+            end
+            cookies
+        end
+
+        def clear_cookies
+            @jar = ::CookieJar::Jar.new
+        end
+    end # CookieJar
+
+    # HTTPS Proxy - connect to proxy
+    # CONNECT #{target_host}:#{target_port} HTTP/1.0\r\n"
+    # Proxy-Authorization: Basic #{encoded_credentials}\r\n
+    # \r\n
+    # Parse response =~ %r{\AHTTP/1\.[01] 200 .*\r\n\r\n}m
+    # use_tls
+    # send requests as usual
+
+    # HTTP Proxy - connect to proxy
+    # GET #{url_with_host} HTTP/1.1\r\n"
+    # Proxy-Authorization: Basic #{encoded_credentials}\r\n
+    # \r\n
+
+    class HttpEndpoint
+        class Connection < OutboundConnection
+            def initialize(host, port, tls, proxy, client)
+                @target_host = host
+                @client = client
+                @request = nil
+
+                if proxy
+                    super(proxy[:host], proxy[:port])
+                    if tls
+                        @negotiating = true
+                        @proxy = proxy
+                        @connect_host = host
+                        @connect_port = port
+                    end
+                else
+                    super(host, port)
+                    start_tls if tls
+                end
+            end
+
+            def start_tls
+                opts = {host_name: @target_host}.merge(@client.tls_options)
+                use_tls(opts)
+            end
+
+            def connect_send_handshake(target_host, target_port, proxy)
+                header = String.new("CONNECT #{target_host}:#{target_port} HTTP/1.0\r\n")
+                if proxy[:username] || proxy[:password]
+                    encoded_credentials = Base64.strict_encode64([proxy[:username], proxy[:password]].join(":"))
+                    header << "Proxy-Authorization: Basic #{encoded_credentials}\r\n"
+                end
+                header << "\r\n"
+                write(header)
+            end
+
+            attr_accessor :request, :reason
+
+            def on_read(data, *args)
+                if @negotiating
+                    @negotiating = false
+                    if data =~ %r{\AHTTP/1\.[01] 200 .*\r\n\r\n}m
+                        start_tls
+                        @client.connection_ready
+                    else
+                        @reason = "Unexpected response from proxy: #{data}"
+                        close_connection
+                    end
+                else
+                    @client.data_received(data)
+                end
+            end
+
+            def post_init(*args)
+            end
+
+            def on_connect(transport)
+                if @negotiating
+                    connect_send_handshake(@connect_host, @connect_port, @proxy)
+                else
+                    @client.connection_ready
+                end
+            end
+
+            def on_close
+                @client.connection_closed(@request, @reason)
+            ensure
+                @request = nil
+                @client = nil
+                @reason = nil
+            end
+
+            def close_connection(request = nil)
+                if request.is_a? Http::Request
+                    @request = request
+                    super(:after_writing)
+                else
+                    super(request)
+                end
+            end
+        end
+
+
+        @@defaults = {
+            :path => '/',
+            :keepalive => true
+        }
+
+
+        def initialize(host, options = {})
+            @queue = []
+            @parser = Http::Parser.new
+            @thread = reactor
+            @connection = nil
+
+            @options = @@defaults.merge(options)
+            @tls_options = options[:tls_options] || {}
+            @inactivity_timeout = options[:inactivity_timeout] || 10000
+
+            uri = host.is_a?(::URI) ? host : ::URI.parse(host)
+            @port = uri.port
+            @host = uri.host
+
+            default_port = uri.port == uri.default_port
+            @encoded_host = default_port ? @host : "#{@host}:#{@port}"
+            @proxy = @options[:proxy]
+
+            @scheme = uri.scheme
+            @tls = @scheme == 'https'
+            @cookiejar = CookieJar.new
+            @middleware = []
+
+            @closing = false
+            @connecting = false
+        end
+
+
+        attr_accessor :inactivity_timeout
+        attr_reader :tls_options, :port, :host, :tls, :scheme, :encoded_host
+        attr_reader :cookiejar, :middleware, :thread, :proxy
+
+
+        def get(options = {});     request(:get,     options); end
+        def head(options = {});    request(:head,    options); end
+        def delete(options = {});  request(:delete,  options); end
+        def put(options = {});     request(:put,     options); end
+        def post(options = {});    request(:post,    options); end
+        def patch(options = {});   request(:patch,   options); end
+        def options(options = {}); request(:options, options); end
+
+
+        def request(method, options = {})
+            options = @options.merge(options)
+            options[:method] = method.to_sym
+
+            # Setup the request with callbacks
+            request = Http::Request.new(self, options)
+            request.then(proc { |response|
+                if response.keep_alive
+                    restart_timer
+                else
+                    # We might have already started processing the next request
+                    # at this point. So don't want to disconnect if already
+                    # disconnected.
+                    close_connection unless @connecting
+                end
+
+                next_request
+
+                response
+            }, proc { |err|
+                # @parser.eof
+                close_connection unless @connecting
+                next_request
+                ::MTLibuv::Q.reject(@thread, err)
+            })
+
+            @queue.unshift(request)
+
+            next_request
+            request
+        end
+
+        # Callbacks
+        def connection_ready
+            # A connection can be closed while still connecting
+            return if @closing
+
+            @connecting = false
+            if @queue.length > 0
+                restart_timer
+                next_request
+            else
+                close_connection
+            end
+        end
+
+        def connection_closed(request, reason)
+            # A connection might close due to a connection failure
+            awaiting_close = @closing
+            awaiting_connect = @connecting
+            @closing = false
+            @connecting = false
+            @connection = nil
+
+            # We may have closed a previous connection
+            if @parser.request && (request.nil? || request == @parser.request)
+                stop_timer
+                @parser.eof
+            elsif request.nil? && @parser.request.nil? && @queue.length > 0
+                req = @queue.pop
+                req.reject(reason || :connection_failure)
+            end
+
+            next_request if awaiting_close || awaiting_connect
+        end
+
+        def data_received(data)
+            restart_timer
+            close_connection if @parser.received(data)
+        end
+
+        def cancel_all
+            @queue.each do |request|
+                request.reject(:cancelled)
+            end
+            if @parser.request
+                @parser.request.reject(:cancelled)
+                @parser.eof
+            end
+            @queue = []
+            close_connection
+        end
+
+        def http_proxy?
+            @proxy && !@tls
+        end
+
+
+        private
+
+
+        def next_request
+            # Don't start a request while transitioning state
+            return if @closing || @connecting
+            return if @parser.request || @queue.length == 0
+
+            if @connection
+                req = @queue.pop
+                @connection.request = req
+                @parser.new_request(req)
+
+                req.execute(@connection)
+            else
+                new_connection
+            end
+        end
+
+        def new_connection
+            # no new connections while transitioning state
+            return if @closing || @connecting
+            if @queue.length > 0 && @connection.nil?
+                @connecting = true
+                @connection = Connection.new(@host, @port, @tls, @proxy, self)
+                start_timer
+            end
+            @connection
+        end
+
+        def close_connection
+            # Close connection can be called while connecting
+            return if @closing || @connection.nil?
+            @closing = true
+            @connection.close_connection
+            stop_timer
+            @connection = nil
+        end
+
+        def start_timer
+            # Only start the timer if there is a connection starting or in place
+            return if @closing || @connection.nil?
+            @timer.cancel if @timer
+            @timer = @thread.scheduler.in(@inactivity_timeout) do
+                @timer = nil
+                idle_timeout
+            end
+        end
+        alias_method :restart_timer, :start_timer
+
+        def stop_timer
+            @timer.cancel unless @timer.nil?
+            @timer = nil
+        end
+
+        def idle_timeout
+            connection = @connection
+            close_connection
+            @parser.reason = :timeout if @parser.request
+            connection.reason = :timeout if connection
+        end
+    end
+end