excon 0.29.0 → 0.30.0

data/tests/middlewares/redirect_follower_tests.rb

@@ -30,3 +30,36 @@ Shindo.tests('Excon redirector support') do
 
   env_restore
 end
+
+Shindo.tests('Excon redirect support for relative Location headers') do
+  env_init
+  
+  connection = Excon.new(
+    'http://127.0.0.1:9292',
+    :middlewares  => Excon.defaults[:middlewares] + [Excon::Middleware::RedirectFollower],
+    :mock         => true
+  )
+
+  Excon.stub(
+    { :path => '/old' },
+    {
+      :headers  => { 'Location' => '/new' },
+      :body     => 'old',
+      :status   => 301
+    }
+  )
+
+  Excon.stub(
+    { :path => '/new' },
+    {
+      :body     => 'new',
+      :status   => 200
+    }
+  )
+
+  tests("request(:method => :get, :path => '/old').body").returns('new') do
+    connection.request(:method => :get, :path => '/old').body
+  end
+  
+  env_restore
+end

data/tests/requests_tests.rb

@@ -42,4 +42,32 @@ Shindo.tests('requests should succeed') do
       end
     end
   end
+
+  with_server('good') do
+
+    tests('sets transfer-coding and connection options') do
+
+      tests('without a :response_block') do
+        request = Marshal.load(
+          Excon.get('http://127.0.0.1:9292/echo/request').body
+        )
+        returns('trailers, deflate, gzip') { request[:headers]['TE'] }
+        returns('TE') { request[:headers]['Connection'] }
+      end
+
+      tests('with a :response_block') do
+        captures = capture_response_block do |block|
+          Excon.get('http://127.0.0.1:9292/echo/request',
+                    :response_block => block)
+        end
+        data = captures.map {|capture| capture[0] }.join
+        request = Marshal.load(data)
+
+        returns('trailers') { request[:headers]['TE'] }
+        returns('TE') { request[:headers]['Connection'] }
+      end
+
+    end
+
+  end
 end

data/tests/response_tests.rb

@@ -0,0 +1,220 @@
+Shindo.tests('Excon Response Parsing') do
+  env_init
+
+  with_server('good') do
+
+    tests('responses with chunked transfer-encoding') do
+
+      tests('simple response').returns('hello world') do
+        Excon.get('http://127.0.0.1:9292/chunked/simple').body
+      end
+
+      tests('with :response_block') do
+
+        tests('simple response').
+              returns([['hello ', nil, nil], ['world', nil, nil]]) do
+          capture_response_block do |block|
+            Excon.get('http://127.0.0.1:9292/chunked/simple',
+                      :response_block => block,
+                      :chunk_size => 5) # not used
+          end
+        end
+
+        tests('with expected response status').
+              returns([['hello ', nil, nil], ['world', nil, nil]]) do
+          capture_response_block do |block|
+            Excon.get('http://127.0.0.1:9292/chunked/simple',
+                      :response_block => block,
+                      :expects => 200)
+          end
+        end
+
+        tests('with unexpected response status').returns('hello world') do
+          begin
+            Excon.get('http://127.0.0.1:9292/chunked/simple',
+                      :response_block => Proc.new { raise 'test failed' },
+                      :expects => 500)
+          rescue Excon::Errors::HTTPStatusError => err
+            err.response[:body]
+          end
+        end
+
+      end
+
+      tests('merges trailers into headers').
+          returns('one, two, three, four, five, six') do
+        Excon.get('http://127.0.0.1:9292/chunked/trailers').headers['Test-Header']
+      end
+
+      tests("removes 'chunked' from Transfer-Encoding").returns('') do
+        Excon.get('http://127.0.0.1:9292/chunked/simple').headers['Transfer-Encoding']
+      end
+
+    end
+
+    tests('responses with content-length') do
+
+      tests('simple response').returns('hello world') do
+        Excon.get('http://127.0.0.1:9292/content-length/simple').body
+      end
+
+      tests('with :response_block') do
+
+        tests('simple response').
+              returns([['hello', 6, 11], [' worl', 1, 11], ['d', 0, 11]]) do
+          capture_response_block do |block|
+            Excon.get('http://127.0.0.1:9292/content-length/simple',
+                      :response_block => block,
+                      :chunk_size => 5)
+          end
+        end
+
+        tests('with expected response status').
+              returns([['hello', 6, 11], [' worl', 1, 11], ['d', 0, 11]]) do
+          capture_response_block do |block|
+            Excon.get('http://127.0.0.1:9292/content-length/simple',
+                      :response_block => block,
+                      :chunk_size => 5,
+                      :expects => 200)
+          end
+        end
+
+        tests('with unexpected response status').returns('hello world') do
+          begin
+            Excon.get('http://127.0.0.1:9292/content-length/simple',
+                      :response_block => Proc.new { raise 'test failed' },
+                      :chunk_size => 5,
+                      :expects => 500)
+          rescue Excon::Errors::HTTPStatusError => err
+            err.response[:body]
+          end
+        end
+
+      end
+
+    end
+
+    tests('responses with unknown length') do
+
+      tests('simple response').returns('hello world') do
+        Excon.get('http://127.0.0.1:9292/unknown/simple').body
+      end
+
+      tests('with :response_block') do
+
+        tests('simple response').
+              returns([['hello', nil, nil], [' worl', nil, nil], ['d', nil, nil]]) do
+          capture_response_block do |block|
+            Excon.get('http://127.0.0.1:9292/unknown/simple',
+                      :response_block => block,
+                      :chunk_size => 5)
+          end
+        end
+
+        tests('with expected response status').
+              returns([['hello', nil, nil], [' worl', nil, nil], ['d', nil, nil]]) do
+          capture_response_block do |block|
+            Excon.get('http://127.0.0.1:9292/unknown/simple',
+                      :response_block => block,
+                      :chunk_size => 5,
+                      :expects => 200)
+          end
+        end
+
+        tests('with unexpected response status').returns('hello world') do
+          begin
+            Excon.get('http://127.0.0.1:9292/unknown/simple',
+                      :response_block => Proc.new { raise 'test failed' },
+                      :chunk_size => 5,
+                      :expects => 500)
+          rescue Excon::Errors::HTTPStatusError => err
+            err.response[:body]
+          end
+        end
+
+      end
+
+    end
+
+    tests('header continuation') do
+
+      tests('proper continuation').returns('one, two, three, four, five, six') do
+        resp = Excon.get('http://127.0.0.1:9292/unknown/header_continuation')
+        resp.headers['Test-Header']
+      end
+
+      tests('malformed header').raises(Excon::Errors::SocketError) do
+        Excon.get('http://127.0.0.1:9292/bad/malformed_header')
+      end
+
+      tests('malformed header continuation').raises(Excon::Errors::SocketError) do
+        Excon.get('http://127.0.0.1:9292/bad/malformed_header_continuation')
+      end
+
+    end
+
+    tests('Transfer-Encoding') do
+
+      tests('used with chunked response') do
+        resp = Excon.post(
+          'http://127.0.0.1:9292/echo/transfer-encoded/chunked',
+          :body => 'hello world'
+        )
+
+        tests('server sent transfer-encoding').returns('gzip, chunked') do
+          resp[:headers]['Transfer-Encoding-Sent']
+        end
+
+        tests('processed encodings removed from header').returns('') do
+          resp[:headers]['Transfer-Encoding']
+        end
+
+        tests('response body decompressed').returns('hello world') do
+          resp[:body]
+        end
+      end
+
+      tests('used with non-chunked response') do
+        resp = Excon.post(
+          'http://127.0.0.1:9292/echo/transfer-encoded',
+          :body => 'hello world'
+        )
+
+        tests('server sent transfer-encoding').returns('gzip') do
+          resp[:headers]['Transfer-Encoding-Sent']
+        end
+
+        tests('processed encoding removed from header').returns('') do
+          resp[:headers]['Transfer-Encoding']
+        end
+
+        tests('response body decompressed').returns('hello world') do
+          resp[:body]
+        end
+      end
+
+      # sends TE header without gzip/deflate accepted (see requests_tests)
+      tests('with a :response_block') do
+        resp = nil
+        captures = capture_response_block do |block|
+          resp = Excon.post('http://127.0.0.1:9292/echo/transfer-encoded/chunked',
+                            :body => 'hello world',
+                            :response_block => block)
+        end
+
+        tests('server does not compress').returns('chunked') do
+          resp[:headers]['Transfer-Encoding-Sent']
+        end
+
+        tests('block receives uncompressed response').returns('hello world') do
+          captures.map {|capture| capture[0] }.join
+        end
+
+      end
+
+    end
+
+  end
+
+  env_restore
+end

data/tests/servers/good.rb

@@ -0,0 +1,313 @@
+#!/usr/bin/env ruby
+
+require 'eventmachine'
+require 'stringio'
+require 'uri'
+require 'zlib'
+
+module GoodServer
+  # This method will be called with each request received.
+  #
+  #     request = {
+  #       :method   => method,
+  #       :uri      => URI.parse(uri),
+  #       :headers  => {},
+  #       :body     => ''
+  #     }
+  #
+  # Each connection to this server is persistent unless the client sends
+  # "Connection: close" in the request. If a response requires the connection
+  # to be closed, it should set `@persistent = false` and send "Connection: close".
+  def send_response(request)
+    type, path = request[:uri].path.split('/', 3)[1, 2]
+    case type
+    when 'echo'
+      case path
+      when 'request'
+        data = Marshal.dump(request)
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Content-Length: #{ data.size }\r\n"
+        send_data "\r\n"
+        send_data data
+
+      when /(content|transfer)-encoded\/?(.*)/
+        if (encoding_type = $1) == 'content'
+          accept_header = 'Accept-Encoding'
+          encoding_header = 'Content-Encoding'
+        else
+          accept_header = 'TE'
+          encoding_header = 'Transfer-Encoding'
+        end
+        chunked = $2 == 'chunked'
+
+        encodings = parse_encodings(request[:headers][accept_header])
+        while encoding = encodings.pop
+          break if ['gzip', 'deflate'].include?(encoding)
+        end
+
+        case encoding
+        when 'gzip'
+          io = (Zlib::GzipWriter.new(StringIO.new) << request[:body]).finish
+          io.rewind
+          body = io.read
+        when 'deflate'
+          # drops the zlib header
+          deflator = Zlib::Deflate.new(nil, -Zlib::MAX_WBITS)
+          body = deflator.deflate(request[:body], Zlib::FINISH)
+          deflator.close
+        else
+          body = request[:body]
+        end
+
+        # simulate server pre/post content encoding
+        encodings = [
+          request[:headers]["#{ encoding_header }-Pre"],
+          encoding,
+          request[:headers]["#{ encoding_header }-Post"],
+        ]
+        if chunked && encoding_type == 'transfer'
+          encodings << 'chunked'
+        end
+        encodings = encodings.compact.join(', ')
+
+        send_data "HTTP/1.1 200 OK\r\n"
+        # let the test know what the server sent
+        send_data "#{ encoding_header }-Sent: #{ encodings }\r\n"
+        send_data "#{ encoding_header }: #{ encodings }\r\n" unless encodings.empty?
+        if chunked
+          if encoding_type == 'content'
+            send_data "Transfer-Encoding: chunked\r\n"
+          end
+          send_data "\r\n"
+          send_data chunks_for(body)
+          send_data "\r\n"
+        else
+          send_data "Content-Length: #{ body.size }\r\n"
+          send_data "\r\n"
+          send_data body
+        end
+      end
+
+    when 'chunked'
+      case path
+      when 'simple'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Transfer-Encoding: chunked\r\n"
+        send_data "\r\n"
+        # chunk-extension is currently ignored.
+        # this works because "6; chunk-extension".to_i => "6"
+        send_data "6; chunk-extension\r\n"
+        send_data "hello \r\n"
+        send_data "5; chunk-extension\r\n"
+        send_data "world\r\n"
+        send_data "0; chunk-extension\r\n" # last-chunk
+        send_data "\r\n"
+
+      # merged trailers also support continuations
+      when 'trailers'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Transfer-Encoding: chunked\r\n"
+        send_data "Test-Header: one, two\r\n"
+        send_data "\r\n"
+        send_data chunks_for('hello world')
+        send_data "Test-Header: three, four,\r\n"
+        send_data "\tfive, six\r\n"
+        send_data "\r\n"
+      end
+
+    when 'content-length'
+      case path
+      when 'simple'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Content-Length: 11\r\n"
+        send_data "\r\n"
+        send_data "hello world"
+      end
+
+    when 'unknown'
+      @persistent = false
+      case path
+      when 'simple'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Connection: close\r\n"
+        send_data "\r\n"
+        send_data "hello world"
+
+      when 'header_continuation'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Connection: close\r\n"
+        send_data "Test-Header: one, two\r\n"
+        send_data "Test-Header: three, four,\r\n"
+        send_data "  five, six\r\n"
+        send_data "\r\n"
+        send_data "hello world"
+      end
+
+    when 'bad'
+      # Excon will close these connections due to the errors.
+      case path
+      when 'malformed_header'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data "Bad-Header\r\n"  # no ':'
+        send_data "\r\n"
+        send_data "hello world"
+
+      when 'malformed_header_continuation'
+        send_data "HTTP/1.1 200 OK\r\n"
+        send_data " Bad-Header: one, two\r\n"  # no previous header
+        send_data "\r\n"
+        send_data "hello world"
+      end
+    end
+
+    close_connection(true) unless @persistent
+  end
+
+  def post_init
+    @buffer = StringIO.new
+    @buffer.set_encoding('BINARY') if @buffer.respond_to?(:set_encoding)
+  end
+
+  # Receives a String of +data+ sent from the client.
+  # +data+ may only be a portion of what the client sent.
+  # The data is buffered, then processed and removed from the buffer
+  # as data becomes available until the @request is complete.
+  def receive_data(data)
+    @buffer.write(data)
+
+    parse_headers unless @request
+    parse_body if @request
+
+    if @request_complete
+      send_response(@request)
+      sync_buffer
+      @request = nil
+      @request_complete = false
+    end
+
+    @buffer.seek(0, IO::SEEK_END)  # wait for more data
+  end
+
+  # Removes the processed portion of the buffer
+  # by replacing the buffer with it's contents from the current pos.
+  def sync_buffer
+    @buffer.string = @buffer.read
+  end
+
+  def parse_headers
+    @buffer.rewind
+    # wait until buffer contains the end of the headers
+    if /\sHTTP\/\d+\.\d+\r\n.*?\r\n\r\n/m =~ @buffer.read
+      @buffer.rewind
+      # For persistent connections, the buffer could start with the
+      # \r\n chunked-message terminator from the previous request.
+      # This will discard anything up to the request-line.
+      until m = /^(\w+)\s(.*)\sHTTP\/\d+\.\d+$/.match(@buffer.readline.chop!); end
+      method, uri = m[1, 2]
+
+      headers = {}
+      last_key = nil
+      until (line = @buffer.readline.chop!).empty?
+        if !line.lstrip!.nil?
+          headers[last_key] << ' ' << line.rstrip
+        else
+          key, value = line.split(':', 2)
+          headers[key] = ([headers[key]] << value.strip).compact.join(', ')
+          last_key = key
+        end
+      end
+
+      sync_buffer
+
+      @chunked = headers['Transfer-Encoding'] =~ /chunked/i
+      @content_length = headers['Content-Length'].to_i
+      @persistent = headers['Connection'] !~ /close/i
+      @request = {
+        :method   => method,
+        :uri      => URI.parse(uri),
+        :headers  => headers,
+        :body     => ''
+      }
+    end
+  end
+
+  def parse_body
+    if @chunked
+      @buffer.rewind
+      until @request_complete || @buffer.eof?
+        unless @chunk_size
+          # in case buffer only contains a portion of the chunk-size line
+          if (line = @buffer.readline) =~ /\r\n\z/
+            @chunk_size = line.to_i(16)
+            if @chunk_size > 0
+              sync_buffer
+            else # last-chunk
+              @buffer.read(2)  # the final \r\n may or may not be in the buffer
+              @chunk_size = nil
+              @body_pos = nil
+              @request_complete = true
+            end
+          end
+        end
+        if @chunk_size
+          if @buffer.size >= @chunk_size + 2
+            @request[:body] << @buffer.read(@chunk_size + 2).chop!
+            @chunk_size = nil
+            sync_buffer
+          else
+            break # wait for more data
+          end
+        end
+      end
+    elsif @content_length > 0
+      @buffer.rewind
+      unless @buffer.eof?  # buffer only contained the headers
+        @request[:body] << @buffer.read(@content_length - @request[:body].size)
+        if @request[:body].size == @content_length
+          @request_complete = true
+        else
+          sync_buffer
+        end
+      end
+    else
+      # no body
+      @request_complete = true
+    end
+  end
+
+  def chunks_for(str)
+    chunks = ''
+    str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
+    chunk_size = str.size / 2
+    until (chunk = str.slice!(0, chunk_size)).empty?
+      chunks << chunk.size.to_s(16) << "\r\n"
+      chunks << chunk << "\r\n"
+    end
+    chunks << "0\r\n" # last-chunk
+  end
+
+  # only supports a single quality parameter for tokens
+  def parse_encodings(encodings)
+    return [] if encodings.nil?
+    split_header_value(encodings).map do |value|
+      token, q_val = /^(.*?)(?:;q=(.*))?$/.match(value.strip)[1, 2]
+      if q_val && q_val.to_f == 0
+        nil
+      else
+        [token, (q_val || 1).to_f]
+      end
+    end.compact.sort_by {|_, q_val| q_val }.map {|token, _| token }
+  end
+
+  # Splits a header value +str+ according to HTTP specification.
+  def split_header_value(str)
+    return [] if str.nil?
+    str.strip.scan(%r'\G((?:"(?:\\.|[^"])+?"|[^",]+)+)
+                        (?:,\s*|\Z)'xn).flatten
+  end
+end
+
+EM.run do
+  EM.start_server("127.0.0.1", 9292, GoodServer)
+  $stderr.puts "ready"
+end