em-http-request 1.1.1 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a80a1bc2ca4e1dda7527f249949cb056b0beb886
-  data.tar.gz: 4819cb821fa74543d537416994fbd65b0e0f676c
+SHA256:
+  metadata.gz: 2f4f8a6c2da0b0c94da02566ece7c1d21b8eb0215a8c9cda13fe9c83ac8ee9b3
+  data.tar.gz: 674236e422a3e242e3e3ac32383b048fe12932392ceb21134a990a4f7c8302ff
 SHA512:
-  metadata.gz: b7c257493787ea81e1c1b26287aa88e4e98eba7278fa4734bdf4efe136ef1184a9f7d464dc821f35775535c649204b2331a2518bdc02e92167e062e8d5f74919
-  data.tar.gz: 1e3f32f26b6bee0e85a11318e1cce1df5fbb9a1c5122e04e9d0610b18e2ce82ec92e04d33482d8981964b33fa36fb61090e8915be3027a86608166997b9549a3
+  metadata.gz: da7d9c33b2275b7974b8fbe0aa8dabda4a675b72e589764afc4af0b19831d566066884368b61c0a18f72509b8b8f98bec87681568deaaa5791a692d323f3c5ac
+  data.tar.gz: c2ebbf1c4ddfc08b9d7fc708b604cf1a95c342f09433959ab4ea5018e0704c0f68bc81004244b24f233993161e6453607d655e88634cd42eea0ce37cc9e9f554

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.2.4
+script: bundle exec rake spec
+notifications:
+  email:
+    - ilya@igvita.com

data/README.md

@@ -1,4 +1,6 @@
-# EM-HTTP-Request
+# EM-HTTP-Request 
+
+[![Gem Version](https://badge.fury.io/rb/em-http-request.svg)](http://rubygems.org/gems/em-http-request) [![Build Status](https://travis-ci.org/igrigorik/em-http-request.svg)](https://travis-ci.org/igrigorik/em-http-request)
 
 Async (EventMachine) HTTP client, with support for:
 
@@ -10,7 +12,7 @@ Async (EventMachine) HTTP client, with support for:
 - Streaming file uploads
 - HTTP proxy and SOCKS5 support
 - Basic Auth & OAuth
-- Connection-level & Global middleware support
+- Connection-level & global middleware support
 - HTTP parser via [http_parser.rb](https://github.com/tmm1/http_parser.rb)
 - Works wherever EventMachine runs: Rubinius, JRuby, MRI
 
@@ -18,7 +20,7 @@ Async (EventMachine) HTTP client, with support for:
 
     gem install em-http-request
 
-- Introductory [screencast](http://everburning.com/news/eventmachine-screencast-em-http-request/)
+- Introductory [screencast](http://everburning.com/news/eventmachine-screencast-em-http-request)
 - [Issuing GET/POST/etc requests](https://github.com/igrigorik/em-http-request/wiki/Issuing-Requests)
 - [Issuing parallel requests with Multi interface](https://github.com/igrigorik/em-http-request/wiki/Parallel-Requests)
 - [Handling Redirects & Timeouts](https://github.com/igrigorik/em-http-request/wiki/Redirects-and-Timeouts)
@@ -54,7 +56,6 @@ Several higher-order Ruby projects have incorporated em-http and other Ruby HTTP
 - [Firering](https://github.com/EmmanuelOga/firering) - Eventmachine powered Campfire API
 - [RDaneel](https://github.com/hasmanydevelopers/RDaneel) - Ruby crawler which respects robots.txt
 - [em-eventsource](https://github.com/AF83/em-eventsource) - EventSource client for EventMachine
-- [sinatra-synchrony](https://github.com/kyledrake/sinatra-synchrony) - Sinatra plugin for synchronous use of EM
 - and many others.. drop me a link if you want yours included!
 
 ### License

data/em-http-request.gemspec

@@ -12,17 +12,18 @@ Gem::Specification.new do |s|
   s.homepage    = 'http://github.com/igrigorik/em-http-request'
   s.summary     = 'EventMachine based, async HTTP Request client'
   s.description = s.summary
+  s.license     = 'MIT'
   s.rubyforge_project = 'em-http-request'
 
   s.add_dependency 'addressable', '>= 2.3.4'
-  s.add_dependency 'cookiejar'
+  s.add_dependency 'cookiejar', '!= 0.3.1'
   s.add_dependency 'em-socksify', '>= 0.3'
   s.add_dependency 'eventmachine', '>= 1.0.3'
-  s.add_dependency 'http_parser.rb', '>= 0.6.0.beta.2'
+  s.add_dependency 'http_parser.rb', '>= 0.6.0'
 
   s.add_development_dependency 'mongrel', '~> 1.2.0.pre2'
   s.add_development_dependency 'multi_json'
-  s.add_development_dependency 'rack'
+  s.add_development_dependency 'rack', '< 2.0'
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rspec'
 

data/lib/em-http.rb

@@ -5,6 +5,7 @@ require 'http/parser'
 
 require 'base64'
 require 'socket'
+require 'openssl'
 
 require 'em-http/core_ext/bytesize'
 require 'em-http/http_connection'

data/lib/em-http/client.rb

@@ -21,7 +21,7 @@ module EventMachine
 
     CRLF="\r\n"
 
-    attr_accessor :state, :response
+    attr_accessor :state, :response, :conn
     attr_reader   :response_header, :error, :content_charset, :req, :cookies
 
     def initialize(conn, options)
@@ -100,14 +100,13 @@ module EventMachine
 
               @cookies.clear
               @cookies = @cookiejar.get(@response_header.location).map(&:to_s) if @req.pass_cookies
-              @req.set_uri(@response_header.location)
 
-              @conn.redirect(self)
+              @conn.redirect(self, @response_header.location)
             else
               succeed(self)
             end
 
-          rescue Exception => e
+          rescue => e
             on_error(e.message)
           end
         else
@@ -156,11 +155,16 @@ module EventMachine
 
       # Set the User-Agent if it hasn't been specified
       if !head.key?('user-agent')
-        head['user-agent'] = "EventMachine HttpClient"
+        head['user-agent'] = 'EventMachine HttpClient'
       elsif head['user-agent'].nil?
         head.delete('user-agent')
       end
 
+      # Set the Accept-Encoding header if none is provided
+      if !head.key?('accept-encoding') && req.compressed
+        head['accept-encoding'] = 'gzip, compressed'
+      end
+
       # Set the auth from the URI if given
       head['Authorization'] = @req.uri.userinfo.split(/:/, 2) if @req.uri.userinfo
 
@@ -178,7 +182,7 @@ module EventMachine
       # Set the Content-Length if body is given,
       # or we're doing an empty post or put
       if body
-        head['content-length'] = body.bytesize
+        head['content-length'] ||= body.respond_to?(:bytesize) ? body.bytesize : body.size
       elsif @req.method == 'POST' or @req.method == 'PUT'
         # wont happen if body is set and we already set content-length above
         head['content-length'] ||= 0
@@ -189,16 +193,13 @@ module EventMachine
         head['content-type'] = 'application/x-www-form-urlencoded'
       end
 
-      request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts.proxy)
+      request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts)
       request_header << encode_headers(head)
       request_header << CRLF
       @conn.send_data request_header
 
-      if body
-        @conn.send_data body
-      elsif @req.file
-        @conn.stream_file_data @req.file, :http_chunks => false
-      end
+      @req_body = body || (@req.file && Pathname.new(@req.file))
+      send_request_body unless @req.headers['expect'] == '100-continue'
     end
 
     def on_body_data(data)
@@ -222,6 +223,28 @@ module EventMachine
       end
     end
 
+    def request_body_pending?
+      !!@req_body
+    end
+
+    def send_request_body
+      return  if @req_body.nil?
+
+      if @req_body.is_a?(String)
+        @conn.send_data @req_body
+
+      elsif @req_body.is_a?(Pathname)
+        @conn.stream_file_data @req_body.to_path, http_chunks: false
+
+      elsif @req_body.respond_to?(:read) && @req_body.respond_to?(:eof?)   # IO or IO-like object
+        @conn.stream_data @req_body
+
+      else
+        raise "Don't know how to send request body: #{@req_body.inspect}"
+      end
+      @req_body = nil
+    end
+
     def parse_response_header(header, version, status)
       @response_header.raw = header
       header.each do |key, val|

data/lib/em-http/decoders.rb

@@ -127,7 +127,6 @@ module EventMachine::HttpDecoders
 
     def extract_stream(compressed)
       @data << compressed
-      pos = @pos
 
       while !eof? && !finished?
         buffer = ""
@@ -208,7 +207,7 @@ module EventMachine::HttpDecoders
       end
 
       if finished?
-        compressed[(@pos - pos)..-1]
+        compressed[(@pos - (@data.length - compressed.length))..-1]
       else
         ""
       end

data/lib/em-http/http_client_options.rb

@@ -1,7 +1,7 @@
 class HttpClientOptions
   attr_reader :uri, :method, :host, :port
   attr_reader :headers, :file, :body, :query, :path
-  attr_reader :keepalive, :pass_cookies, :decoding
+  attr_reader :keepalive, :pass_cookies, :decoding, :compressed
 
   attr_accessor :followed, :redirects
 
@@ -12,40 +12,38 @@ class HttpClientOptions
 
     @method   = method.to_s.upcase
     @headers  = options[:head] || {}
-    @query    = options[:query]
-
 
     @file     = options[:file]
     @body     = options[:body]
 
     @pass_cookies = options.fetch(:pass_cookies, true)  # pass cookies between redirects
     @decoding     = options.fetch(:decoding, true)      # auto-decode compressed response
+    @compressed   = options.fetch(:compressed, true)    # auto-negotiated compressed response
 
-    set_uri(uri, options[:path])
+    set_uri(uri, options[:path], options[:query])
   end
 
   def follow_redirect?; @followed < @redirects; end
   def ssl?; @uri.scheme == "https" || @uri.port == 443; end
   def no_body?; @method == "HEAD"; end
 
-  def set_uri(uri, path = nil)
+  def set_uri(uri, path = nil, query = nil)
     uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
     uri.path = path if path
     uri.path = '/' if uri.path.empty?
 
     @uri = uri
     @path = uri.path
+    @host = uri.hostname
+    @port = uri.port
+    @query = query
 
     # Make sure the ports are set as Addressable::URI doesn't
     # set the port if it isn't there
-    if @uri.scheme == "https"
-      @uri.port ||= 443
-    else
-      @uri.port ||= 80
+    if @port.nil?
+      @port = @uri.scheme == "https" ? 443 : 80
     end
 
-    @host = @uri.host
-    @port = @uri.port
-
+    uri
   end
 end

data/lib/em-http/http_connection.rb

@@ -1,3 +1,5 @@
+require 'em/io_streamer'
+
 module EventMachine
 
   module HTTPMethods
@@ -20,7 +22,11 @@ module EventMachine
     end
 
     def receive_data(data)
-      @parent.receive_data data
+      begin
+        @parent.receive_data data
+      rescue EventMachine::Connectify::CONNECTError => e
+        @parent.close(e.message)
+      end
     end
 
     def connection_completed
@@ -30,6 +36,62 @@ module EventMachine
     def unbind(reason=nil)
       @parent.unbind(reason)
     end
+
+    # TLS verification support, original implementation by Mislav Marohnić
+    # https://github.com/lostisland/faraday/blob/63cf47c95b573539f047c729bd9ad67560bc83ff/lib/faraday/adapter/em_http_ssl_patch.rb
+    def ssl_verify_peer(cert_string)
+      cert = nil
+      begin
+        cert = OpenSSL::X509::Certificate.new(cert_string)
+      rescue OpenSSL::X509::CertificateError
+        return false
+      end
+
+      @last_seen_cert = cert
+
+      if certificate_store.verify(@last_seen_cert)
+        begin
+          certificate_store.add_cert(@last_seen_cert)
+        rescue OpenSSL::X509::StoreError => e
+          raise e unless e.message == 'cert already in hash table'
+        end
+        true
+      else
+        raise OpenSSL::SSL::SSLError.new(%(unable to verify the server certificate for "#{host}"))
+      end
+    end
+
+    def ssl_handshake_completed
+      unless verify_peer?
+        warn "[WARNING; em-http-request] TLS hostname validation is disabled (use 'tls: {verify_peer: true}'), see" +
+             " CVE-2020-13482 and https://github.com/igrigorik/em-http-request/issues/339 for details"
+        return true
+      end
+
+      unless OpenSSL::SSL.verify_certificate_identity(@last_seen_cert, host)
+        raise OpenSSL::SSL::SSLError.new(%(host "#{host}" does not match the server certificate))
+      else
+        true
+      end
+    end
+
+    def verify_peer?
+      parent.connopts.tls[:verify_peer]
+    end
+
+    def host
+      parent.connopts.host
+    end
+
+    def certificate_store
+      @certificate_store ||= begin
+        store = OpenSSL::X509::Store.new
+        store.set_default_paths
+        ca_file = parent.connopts.tls[:cert_chain_file]
+        store.add_file(ca_file) if ca_file
+        store
+      end
+    end
   end
 
   class HttpConnection
@@ -37,8 +99,8 @@ module EventMachine
     include Socksify
     include Connectify
 
-    attr_reader :deferred
-    attr_accessor :error, :connopts, :uri, :conn
+    attr_reader :deferred, :conn
+    attr_accessor :error, :connopts, :uri
 
     def initialize
       @deferred = true
@@ -97,7 +159,7 @@ module EventMachine
       @conn.callback { c.connection_completed }
 
       middleware.each do |m|
-        c.callback &m.method(:response) if m.respond_to?(:response)
+        c.callback(&m.method(:response)) if m.respond_to?(:response)
       end
 
       @clients.push c
@@ -114,8 +176,21 @@ module EventMachine
       @p = Http::Parser.new
       @p.header_value_type = :mixed
       @p.on_headers_complete = proc do |h|
-        client.parse_response_header(h, @p.http_version, @p.status_code)
-        :reset if client.req.no_body?
+        if client
+          if @p.status_code == 100
+            client.send_request_body
+            @p.reset!
+          else
+            client.parse_response_header(h, @p.http_version, @p.status_code)
+            :reset if client.req.no_body?
+          end
+        else
+          # if we receive unexpected data without a pending client request
+          # reset the parser to avoid firing any further callbacks and close
+          # the connection because we're processing invalid HTTP
+          @p.reset!
+          unbind
+        end
       end
 
       @p.on_body = proc do |b|
@@ -152,9 +227,9 @@ module EventMachine
       @peer = @conn.get_peername
 
       if @connopts.socks_proxy?
-        socksify(client.req.uri.host, client.req.uri.port, *@connopts.proxy[:authorization]) { start }
+        socksify(client.req.uri.hostname, client.req.uri.inferred_port, *@connopts.proxy[:authorization]) { start }
       elsif @connopts.connect_proxy?
-        connectify(client.req.uri.host, client.req.uri.port, *@connopts.proxy[:authorization]) { start }
+        connectify(client.req.uri.hostname, client.req.uri.inferred_port, *@connopts.proxy[:authorization]) { start }
       else
         start
       end
@@ -165,8 +240,33 @@ module EventMachine
       @conn.succeed
     end
 
-    def redirect(client)
-      @pending.push client
+    def redirect(client, new_location)
+      old_location = client.req.uri
+      new_location = client.req.set_uri(new_location)
+
+      if client.req.keepalive
+        # Application requested a keep-alive connection but one of the requests
+        # hits a cross-origin redirect. We need to open a new connection and
+        # let both connections proceed simultaneously.
+        if old_location.origin != new_location.origin
+          conn = HttpConnection.new
+          client.conn = conn
+          conn.connopts = @connopts
+          conn.connopts.https = new_location.scheme == "https"
+          conn.uri = client.req.uri
+          conn.activate_connection(client)
+
+        # If the redirect is a same-origin redirect on a keep-alive request
+        # then immidiately dispatch the request over existing connection.
+        else
+          @clients.push client
+          client.connection_completed
+        end
+      else
+        # If connection is not keep-alive the unbind will fire and we'll
+        # reconnect using the same connection object.
+        @pending.push client
+      end
     end
 
     def unbind(reason = nil)
@@ -208,6 +308,10 @@ module EventMachine
       @conn.stream_file_data filename, args
     end
 
+    def stream_data(io, opts = {})
+      EventMachine::IOStreamer.new(self, io, opts)
+    end
+
     private
 
       def client

data/lib/em-http/http_connection_options.rb

@@ -1,13 +1,13 @@
 class HttpConnectionOptions
   attr_reader :host, :port, :tls, :proxy, :bind, :bind_port
   attr_reader :connect_timeout, :inactivity_timeout
+  attr_writer :https
 
   def initialize(uri, options)
     @connect_timeout     = options[:connect_timeout] || 5        # default connection setup timeout
     @inactivity_timeout  = options[:inactivity_timeout] ||= 10   # default connection inactivity (post-setup) timeout
 
     @tls   = options[:tls] || options[:ssl] || {}
-    @proxy = options[:proxy]
 
     if bind = options[:bind]
       @bind = bind[:host] || '0.0.0.0'
@@ -20,12 +20,15 @@ class HttpConnectionOptions
     uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
     @https = uri.scheme == "https"
     uri.port ||= (@https ? 443 : 80)
+    @tls[:sni_hostname] = uri.hostname
 
-    if proxy = options[:proxy]
+    @proxy = options[:proxy] || proxy_from_env
+
+    if proxy
       @host = proxy[:host]
       @port = proxy[:port]
     else
-      @host = uri.host
+      @host = uri.hostname
       @port = uri.port
     end
   end
@@ -41,4 +44,27 @@ class HttpConnectionOptions
   def socks_proxy?
     @proxy && (@proxy[:type] == :socks5)
   end
+
+  def proxy_from_env
+    # TODO: Add support for $http_no_proxy or $no_proxy ?
+    proxy_str = if @https
+                  ENV['HTTPS_PROXY'] || ENV['https_proxy']
+                else
+                  ENV['HTTP_PROXY'] || ENV['http_proxy']
+
+                # Fall-back to $ALL_PROXY if none of the above env-vars have values
+                end || ENV['ALL_PROXY']
+
+    # Addressable::URI::parse will return `nil` if given `nil` and an empty URL for an empty string
+    # so, let's short-circuit that:
+    return if !proxy_str || proxy_str.empty?
+
+    proxy_env_uri = Addressable::URI::parse(proxy_str)
+    { :host => proxy_env_uri.host, :port => proxy_env_uri.port, :type => :http }
+
+  rescue Addressable::URI::InvalidURIError
+    # An invalid env-var shouldn't crash the config step, IMHO.
+    # We should somehow log / warn about this, perhaps...
+    return
+  end
 end