em-http-request 1.0.2 → 1.0.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of em-http-request might be problematic. Click here for more details.

data/.gitignore CHANGED
@@ -6,3 +6,4 @@ Gemfile.lock
6
6
  misc
7
7
  gems/
8
8
  vendor/
9
+ .idea
@@ -9,7 +9,7 @@ require 'rest_client'
9
9
  require 'tach'
10
10
  require 'typhoeus'
11
11
 
12
- url = 'http://127.0.0.1/10k.html'
12
+ url = 'http://127.0.0.1:9292/data/10000'
13
13
 
14
14
  with_server do
15
15
  Tach.meter(100) do
@@ -119,38 +119,52 @@ with_server do
119
119
  streamly.get(url)
120
120
  end
121
121
 
122
- tach('Typhoeus') do
123
- Typhoeus::Request.get(url).body
122
+ tach('Typhoeus') do |n|
123
+ hydra = Typhoeus::Hydra.new( max_concurrency: 8 )
124
+ hydra.disable_memoization
125
+ count = 0
126
+ error = 0
127
+ n.times {
128
+ req = Typhoeus::Request.new( url )
129
+ req.on_complete do |res|
130
+ count += 1
131
+ error += 1 if !res.success?
132
+ p [count, error] if count == n
133
+
134
+ end
135
+ hydra.queue( req )
136
+ }
137
+ hydra.run
124
138
  end
125
139
 
126
140
  end
127
141
  end
128
142
 
129
143
 
130
- # +------------------------------+----------+
131
- # | tach | total |
132
- # +------------------------------+----------+
133
- # | em-http-request (persistent) | 0.016779 |
134
- # +------------------------------+----------+
135
- # | Excon (persistent) | 0.019606 |
136
- # +------------------------------+----------+
137
- # | curb (persistent) | 0.022034 |
138
- # +------------------------------+----------+
139
- # | Typhoeus | 0.027276 |
140
- # +------------------------------+----------+
141
- # | Excon | 0.034482 |
142
- # +------------------------------+----------+
143
- # | StreamlyFFI (persistent) | 0.036474 |
144
- # +------------------------------+----------+
145
- # | em-http-request | 0.041866 |
146
- # +------------------------------+----------+
147
- # | Net::HTTP (persistent) | 0.098379 |
148
- # +------------------------------+----------+
149
- # | Net::HTTP | 0.103786 |
150
- # +------------------------------+----------+
151
- # | RestClient | 0.111841 |
152
- # +------------------------------+----------+
153
- # | HTTParty | 0.118632 |
154
- # +------------------------------+----------+
155
- # | open-uri | 0.170172 |
156
- # +------------------------------+----------+
144
+ #+------------------------------+-----------+
145
+ #| tach | total |
146
+ #+------------------------------+-----------+
147
+ #| em-http-request (persistent) | 0.145512 |
148
+ #+------------------------------+-----------+
149
+ #| Excon | 0.181564 |
150
+ #+------------------------------+-----------+
151
+ #| RestClient | 0.253127 |
152
+ #+------------------------------+-----------+
153
+ #| Net::HTTP | 0.294412 |
154
+ #+------------------------------+-----------+
155
+ #| HTTParty | 0.305397 |
156
+ #+------------------------------+-----------+
157
+ #| open-uri | 0.307007 |
158
+ #+------------------------------+-----------+
159
+ #| Net::HTTP (persistent) | 0.313716 |
160
+ #+------------------------------+-----------+
161
+ #| Typhoeus | 0.514725 |
162
+ #+------------------------------+-----------+
163
+ #| curb (persistent) | 3.981700 |
164
+ #+------------------------------+-----------+
165
+ #| StreamlyFFI (persistent) | 3.989063 |
166
+ #+------------------------------+-----------+
167
+ #| Excon (persistent) | 4.018761 |
168
+ #+------------------------------+-----------+
169
+ #| em-http-request | 15.025291 |
170
+ #+------------------------------+-----------+
@@ -32,12 +32,12 @@ end
32
32
 
33
33
  def with_server(&block)
34
34
  pid = Process.fork do
35
- # Benchmark::Server.run
35
+ Benchmark::Server.run
36
36
  end
37
37
  loop do
38
38
  sleep(1)
39
39
  begin
40
- # Excon.get('http://localhost:9292/api/foo')
40
+ #Excon.get('http://localhost:9292/api/foo')
41
41
  break
42
42
  rescue
43
43
  end
@@ -45,4 +45,4 @@ def with_server(&block)
45
45
  yield
46
46
  ensure
47
47
  Process.kill(9, pid)
48
- end
48
+ end
@@ -1,33 +1,33 @@
1
- # -*- encoding: utf-8 -*-
2
- $:.push File.expand_path("../lib", __FILE__)
3
- require "em-http/version"
4
-
5
- Gem::Specification.new do |s|
6
- s.name = "em-http-request"
7
- s.version = EventMachine::HttpRequest::VERSION
8
-
9
- s.platform = Gem::Platform::RUBY
10
- s.authors = ["Ilya Grigorik"]
11
- s.email = ["ilya@igvita.com"]
12
- s.homepage = "http://github.com/igrigorik/em-http-request"
13
- s.summary = "EventMachine based, async HTTP Request client"
14
- s.description = s.summary
15
- s.rubyforge_project = "em-http-request"
16
-
17
- s.add_dependency "eventmachine", ">= 1.0.0.beta.4"
18
- s.add_dependency "addressable", ">= 2.2.3"
19
- s.add_dependency "http_parser.rb", ">= 0.5.3"
20
- s.add_dependency "em-socksify"
21
- s.add_dependency "cookiejar"
22
-
23
- s.add_development_dependency "rspec"
24
- s.add_development_dependency "rake"
25
- s.add_development_dependency "rack"
26
- s.add_development_dependency "yajl-ruby"
27
- s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
28
-
29
- s.files = `git ls-files`.split("\n")
30
- s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
31
- s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
32
- s.require_paths = ["lib"]
33
- end
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "em-http/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "em-http-request"
7
+ s.version = EventMachine::HttpRequest::VERSION
8
+
9
+ s.platform = Gem::Platform::RUBY
10
+ s.authors = ["Ilya Grigorik"]
11
+ s.email = ["ilya@igvita.com"]
12
+ s.homepage = "http://github.com/igrigorik/em-http-request"
13
+ s.summary = "EventMachine based, async HTTP Request client"
14
+ s.description = s.summary
15
+ s.rubyforge_project = "em-http-request"
16
+
17
+ s.add_dependency "eventmachine", ">= 1.0.0.beta.4"
18
+ s.add_dependency "addressable", ">= 2.2.3"
19
+ s.add_dependency "http_parser.rb", ">= 0.5.3"
20
+ s.add_dependency "em-socksify"
21
+ s.add_dependency "cookiejar"
22
+
23
+ s.add_development_dependency "rspec"
24
+ s.add_development_dependency "rake"
25
+ s.add_development_dependency "rack"
26
+ s.add_development_dependency "yajl-ruby"
27
+ s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
28
+
29
+ s.files = `git ls-files`.split("\n")
30
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
31
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
32
+ s.require_paths = ["lib"]
33
+ end
@@ -133,9 +133,10 @@ module EventMachine
133
133
 
134
134
  def build_request
135
135
  head = @req.headers ? munge_header_keys(@req.headers) : {}
136
-
137
- if @req.http_proxy?
138
- head['proxy-authorization'] = @req.proxy[:authorization] if @req.proxy[:authorization]
136
+
137
+ if @conn.connopts.http_proxy?
138
+ proxy = @conn.connopts.proxy
139
+ head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
139
140
  end
140
141
 
141
142
  # Set the cookie header if provided
@@ -217,6 +218,7 @@ module EventMachine
217
218
  end
218
219
 
219
220
  def parse_response_header(header, version, status)
221
+ @response_header.raw = header
220
222
  header.each do |key, val|
221
223
  @response_header[key.upcase.gsub('-','_')] = val
222
224
  end
@@ -10,7 +10,7 @@ module EventMachine::HttpDecoders
10
10
 
11
11
  class << self
12
12
  def accepted_encodings
13
- DECODERS.inject([]) { |r,d| r + d.encoding_names }
13
+ DECODERS.inject([]) { |r, d| r + d.encoding_names }
14
14
  end
15
15
 
16
16
  def decoder_for_encoding(encoding)
@@ -44,7 +44,7 @@ module EventMachine::HttpDecoders
44
44
  decompressed = finalize
45
45
  receive_decompressed decompressed
46
46
  end
47
-
47
+
48
48
  private
49
49
 
50
50
  def receive_decompressed(decompressed)
@@ -91,30 +91,52 @@ module EventMachine::HttpDecoders
91
91
  end
92
92
  end
93
93
 
94
- ##
95
- # Oneshot decompressor, due to lack of a streaming Gzip reader
96
- # implementation. We may steal code from Zliby to improve this.
97
- #
98
- # For now, do not put `gzip' or `compressed' in your accept-encoding
99
- # header if you expect much data through the :on_response interface.
100
94
  class GZip < Base
101
95
  def self.encoding_names
102
96
  %w(gzip compressed)
103
97
  end
104
98
 
105
99
  def decompress(compressed)
106
- @buf ||= ''
107
- @buf += compressed
108
- nil
100
+ @buf ||= LazyStringIO.new
101
+ @buf << compressed
102
+
103
+ # Zlib::GzipReader loads input in 2048 byte chunks
104
+ if @buf.size > 2048
105
+ @gzip ||= Zlib::GzipReader.new @buf
106
+ @gzip.readline
107
+ end
109
108
  end
110
109
 
111
110
  def finalize
112
111
  begin
113
- Zlib::GzipReader.new(StringIO.new(@buf.to_s)).read
112
+ @gzip ||= Zlib::GzipReader.new @buf
113
+ @gzip.read
114
114
  rescue Zlib::Error
115
115
  raise DecoderError
116
116
  end
117
117
  end
118
+
119
+ class LazyStringIO
120
+ def initialize(string="")
121
+ @stream = string
122
+ end
123
+
124
+ def <<(string)
125
+ @stream << string
126
+ end
127
+
128
+ def read(length=nil, buffer=nil)
129
+ buffer ||= ""
130
+ length ||= 0
131
+ buffer << @stream[0..(length-1)]
132
+ @stream = @stream[length..-1]
133
+ buffer
134
+ end
135
+
136
+ def size
137
+ @stream.size
138
+ end
139
+ end
118
140
  end
119
141
 
120
142
  DECODERS = [Deflate, GZip]
@@ -1,5 +1,5 @@
1
1
  class HttpClientOptions
2
- attr_reader :uri, :method, :host, :port, :proxy
2
+ attr_reader :uri, :method, :host, :port
3
3
  attr_reader :headers, :file, :body, :query, :path
4
4
  attr_reader :keepalive, :pass_cookies, :decoding
5
5
 
@@ -25,7 +25,6 @@ class HttpClientOptions
25
25
  end
26
26
 
27
27
  def follow_redirect?; @followed < @redirects; end
28
- def http_proxy?; @proxy && [nil, :http].include?(@proxy[:type]); end
29
28
  def ssl?; @uri.scheme == "https" || @uri.port == 443; end
30
29
  def no_body?; @method == "HEAD"; end
31
30
 
@@ -1,11 +1,13 @@
1
1
  module EventMachine
2
2
 
3
3
  module HTTPMethods
4
- def get options = {}, &blk; setup_request(:get, options, &blk); end
5
- def head options = {}, &blk; setup_request(:head, options, &blk); end
6
- def delete options = {}, &blk; setup_request(:delete,options, &blk); end
7
- def put options = {}, &blk; setup_request(:put, options, &blk); end
8
- def post options = {}, &blk; setup_request(:post, options, &blk); end
4
+ def get options = {}, &blk; setup_request(:get, options, &blk); end
5
+ def head options = {}, &blk; setup_request(:head, options, &blk); end
6
+ def delete options = {}, &blk; setup_request(:delete, options, &blk); end
7
+ def put options = {}, &blk; setup_request(:put, options, &blk); end
8
+ def post options = {}, &blk; setup_request(:post, options, &blk); end
9
+ def patch options = {}, &blk; setup_request(:patch, options, &blk); end
10
+ def options options = {}, &blk; setup_request(:options, options, &blk); end
9
11
  end
10
12
 
11
13
  class HttpStubConnection < Connection
@@ -180,10 +182,15 @@ module EventMachine
180
182
  @conn.reconnect(r.req.host, r.req.port)
181
183
  end
182
184
 
185
+ @conn.pending_connect_timeout = @connopts.connect_timeout
186
+ @conn.comm_inactivity_timeout = @connopts.inactivity_timeout
183
187
  @conn.callback { r.connection_completed }
184
188
  rescue EventMachine::ConnectionError => e
185
189
  @clients.pop.close(e.message)
186
190
  end
191
+ else
192
+ @deferred = true
193
+ @conn.close_connection
187
194
  end
188
195
  end
189
196
  alias :close :unbind
@@ -11,7 +11,10 @@ class HttpConnectionOptions
11
11
 
12
12
  if bind = options[:bind]
13
13
  @bind = bind[:host] || '0.0.0.0'
14
- @bind_port = bind[:port] || 0
14
+
15
+ # Eventmachine will open a UNIX socket if bind :port
16
+ # is explicitly set to nil
17
+ @bind_port = bind[:port]
15
18
  end
16
19
 
17
20
  uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
@@ -25,4 +28,6 @@ class HttpConnectionOptions
25
28
  @port = uri.port
26
29
  end
27
30
  end
31
+
32
+ def http_proxy?; @proxy && [nil, :http].include?(@proxy[:type]); end
28
33
  end
@@ -11,6 +11,9 @@ module EventMachine
11
11
  # The status code (as a string!)
12
12
  attr_accessor :http_status
13
13
 
14
+ # Raw headers
15
+ attr_accessor :raw
16
+
14
17
  # E-Tag
15
18
  def etag
16
19
  self[HttpClient::ETAG]
@@ -1,5 +1,5 @@
1
1
  module EventMachine
2
2
  class HttpRequest
3
- VERSION = "1.0.2"
3
+ VERSION = "1.0.3"
4
4
  end
5
5
  end
@@ -1,23 +1,23 @@
1
- require 'helper'
2
- require 'fiber'
3
-
4
- describe EventMachine::HttpRequest do
5
- context "with fibers" do
6
-
7
- it "should be transparent to connection errors" do
8
- EventMachine.run do
9
- Fiber.new do
10
- f = Fiber.current
11
- fired = false
12
- http = EventMachine::HttpRequest.new('http://non-existing.domain/', :connection_timeout => 0.1).get
13
- http.callback { failed(http) }
14
- http.errback { f.resume :errback }
15
-
16
- Fiber.yield.should == :errback
17
- EM.stop
18
- end.resume
19
- end
20
- end
21
-
22
- end
23
- end
1
+ require 'helper'
2
+ require 'fiber'
3
+
4
+ describe EventMachine::HttpRequest do
5
+ context "with fibers" do
6
+
7
+ it "should be transparent to connection errors" do
8
+ EventMachine.run do
9
+ Fiber.new do
10
+ f = Fiber.current
11
+ fired = false
12
+ http = EventMachine::HttpRequest.new('http://non-existing.domain/', :connection_timeout => 0.1).get
13
+ http.callback { failed(http) }
14
+ http.errback { f.resume :errback }
15
+
16
+ Fiber.yield.should == :errback
17
+ EM.stop
18
+ end.resume
19
+ end
20
+ end
21
+
22
+ end
23
+ end
@@ -188,6 +188,19 @@ describe EventMachine::HttpRequest do
188
188
  }
189
189
  end
190
190
 
191
+ it "should perform successful PATCH" do
192
+ EventMachine.run {
193
+ http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').patch :body => "data"
194
+
195
+ http.errback { failed(http) }
196
+ http.callback {
197
+ http.response_header.status.should == 200
198
+ http.response.should match(/data/)
199
+ EventMachine.stop
200
+ }
201
+ }
202
+ end
203
+
191
204
  it "should escape body on POST" do
192
205
  EventMachine.run {
193
206
  http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => {:stuff => 'string&string'}
@@ -214,7 +227,7 @@ describe EventMachine::HttpRequest do
214
227
  }
215
228
  }
216
229
  end
217
-
230
+
218
231
  it "should set content-length to 0 on posts with empty bodies" do
219
232
  EventMachine.run {
220
233
  http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/echo_content_length_from_header').post
@@ -343,6 +356,20 @@ describe EventMachine::HttpRequest do
343
356
  }
344
357
  end
345
358
 
359
+ it "should return raw headers in a hash" do
360
+ EventMachine.run {
361
+ http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/echo_headers').get
362
+
363
+ http.errback { failed(http) }
364
+ http.callback {
365
+ http.response_header.status.should == 200
366
+ http.response_header.raw['Set-Cookie'].should match('test=yes')
367
+ http.response_header.raw['X-Forward-Host'].should match('proxy.local')
368
+ EventMachine.stop
369
+ }
370
+ }
371
+ end
372
+
346
373
  it "should detect deflate encoding" do
347
374
  EventMachine.run {
348
375
 
@@ -362,18 +389,40 @@ describe EventMachine::HttpRequest do
362
389
  it "should detect gzip encoding" do
363
390
  EventMachine.run {
364
391
 
365
- http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip').get :head => {
366
- "accept-encoding" => "gzip, compressed"
367
- }
392
+ http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip').get :head => {"accept-encoding" => "gzip, compressed"}
368
393
 
369
- http.errback { failed(http) }
370
- http.callback {
371
- http.response_header.status.should == 200
372
- http.response_header["CONTENT_ENCODING"].should == "gzip"
373
- http.response.should == "compressed"
394
+ http.errback { failed(http) }
395
+ http.callback {
396
+ http.response_header.status.should == 200
397
+ http.response_header["CONTENT_ENCODING"].should == "gzip"
398
+ http.response.should == "compressed"
374
399
 
375
- EventMachine.stop
400
+ EventMachine.stop
401
+ }
376
402
  }
403
+ end
404
+
405
+ it "should stream gzip responses" do
406
+ expected_response = Zlib::GzipReader.open(File.dirname(__FILE__) + "/fixtures/gzip-sample.gz") { |f| f.read }
407
+ actual_response = ''
408
+
409
+ EventMachine.run {
410
+
411
+ http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip-large').get :head => {"accept-encoding" => "gzip, compressed"}
412
+
413
+ http.errback { failed(http) }
414
+ http.callback {
415
+ http.response_header.status.should == 200
416
+ http.response_header["CONTENT_ENCODING"].should == "gzip"
417
+ http.response.should == ''
418
+
419
+ actual_response.should == expected_response
420
+
421
+ EventMachine.stop
422
+ }
423
+ http.stream do |chunk|
424
+ actual_response << chunk
425
+ end
377
426
  }
378
427
  end
379
428
 
@@ -633,6 +682,9 @@ describe EventMachine::HttpRequest do
633
682
  end
634
683
 
635
684
  it "should get the body without Content-Length" do
685
+ pending "blocked on new http_parser.rb release"
686
+ # https://github.com/igrigorik/em-http-request/issues/168
687
+
636
688
  EventMachine.run {
637
689
  @s = StubServer.new("HTTP/1.1 200 OK\r\n\r\nFoo")
638
690
 
@@ -709,6 +761,24 @@ describe EventMachine::HttpRequest do
709
761
  }
710
762
  end
711
763
 
764
+ it "should reconnect if connection was closed between requests" do
765
+ EventMachine.run {
766
+ conn = EM::HttpRequest.new('http://127.0.0.1:8090/', :inactivity_timeout => 0.5)
767
+ req = conn.get :keepalive => true
768
+
769
+ req.callback do
770
+ EM.add_timer(1) do
771
+ req = conn.get
772
+
773
+ req.callback do
774
+ req.response_header.status.should == 200
775
+ EventMachine.stop
776
+ end
777
+ end
778
+ end
779
+ }
780
+ end
781
+
712
782
  it 'should handle malformed Content-Type header repetitions' do
713
783
  EventMachine.run {
714
784
  response =<<-HTTP.gsub(/^ +/, '').strip