typhoeus 0.6.6 → 0.6.7

Sign up to get free protection for your applications and to get access to all the features.
data/.travis.yml CHANGED
@@ -12,6 +12,9 @@ rvm:
12
12
  - rbx-18mode
13
13
  - rbx-19mode
14
14
  matrix:
15
+ fast_finish: true
15
16
  allow_failures:
16
17
  - rvm: ruby-head
17
18
  - rvm: jruby-head
19
+ - rvm: rbx-18mode
20
+ - rvm: rbx-19mode
data/CHANGELOG.md CHANGED
@@ -4,6 +4,11 @@
4
4
 
5
5
  [Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.6...master)
6
6
 
7
+ Enhancements:
8
+
9
+ * Add response streaming.
10
+ ([\#339](https://github.com/typhoeus/typhoeus/pull/339))
11
+
7
12
  ## 0.6.6
8
13
 
9
14
  [Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.5...v0.6.6)
data/README.md CHANGED
@@ -138,6 +138,31 @@ Typhoeus.post(
138
138
  )
139
139
  ```
140
140
 
141
+ ### Streaming the response body
142
+
143
+ Typhoeus can stream responses. When you're expecting a large response,
144
+ set the `on_body` callback on a request. Typhoeus will yield to the callback
145
+ with chunks of the response, as they're read. When you set an `on_body` callback,
146
+ Typhoeus will not store the complete response.
147
+
148
+ ```ruby
149
+ downloaded_file = File.open 'huge.iso', 'wb'
150
+ request = Typhoeus::Request.new("www.example.com/huge.iso")
151
+ request.on_headers do |response|
152
+ if ! response.success?
153
+ raise "Request failed"
154
+ end
155
+ end
156
+ request.on_body do |chunk|
157
+ downloaded_file.write(chunk)
158
+ end
159
+ request.on_complete do |response|
160
+ downloaded_file.close
161
+ # Note that response.body is ""
162
+ end
163
+ request.run
164
+ ```
165
+
141
166
  ### Making Parallel Requests
142
167
 
143
168
  Generally, you should be running requests through hydra. Here is how that looks
@@ -211,8 +236,8 @@ class Cache
211
236
  memory[request]
212
237
  end
213
238
 
214
- def set(request, reseponse)
215
- memory[request] = reseponse
239
+ def set(request, response)
240
+ memory[request] = response
216
241
  end
217
242
  end
218
243
 
@@ -245,6 +270,16 @@ Typhoeus.get("www.example.com") == response
245
270
  #=> true
246
271
  ```
247
272
 
273
+ When testing make sure to clear your expectations or the stubs will persist between tests. The following can be included in your spec_helper.rb file to do this automatically.
274
+
275
+ ```ruby
276
+ RSpec.configure do |config|
277
+ config.before :each do
278
+ Typhoeus::Expectation.clear
279
+ end
280
+ end
281
+ ```
282
+
248
283
  ### Timeouts
249
284
 
250
285
  No exceptions are raised on HTTP timeouts. You can check whether a request timed out with the following methods:
@@ -368,3 +403,7 @@ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
368
403
  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
369
404
  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
370
405
  OTHER DEALINGS IN THE SOFTWARE.
406
+
407
+
408
+ [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/typhoeus/typhoeus/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
409
+
data/lib/typhoeus.rb CHANGED
@@ -43,7 +43,6 @@ end
43
43
  #
44
44
  # @since 0.5.0
45
45
  module Typhoeus
46
- extend self
47
46
  extend Request::Actions
48
47
  extend Request::Callbacks::Types
49
48
 
@@ -59,7 +58,7 @@ module Typhoeus
59
58
  # @return [ Typhoeus::Config ] The configuration.
60
59
  #
61
60
  # @see Typhoeus::Config
62
- def configure
61
+ def self.configure
63
62
  yield Config
64
63
  end
65
64
 
@@ -73,7 +72,7 @@ module Typhoeus
73
72
  # @return [ Typhoeus::Expectation ] The expecatation.
74
73
  #
75
74
  # @see Typhoeus::Expectation
76
- def stub(base_url, options = {}, &block)
75
+ def self.stub(base_url, options = {}, &block)
77
76
  expectation = Expectation.all.find{ |e| e.base_url == base_url && e.options == options }
78
77
  if expectation.nil?
79
78
  expectation = Expectation.new(base_url, options)
@@ -94,7 +93,7 @@ module Typhoeus
94
93
  # @yield [ Typhoeus::Request ]
95
94
  #
96
95
  # @return [ Array<Block> ] All before blocks.
97
- def before(&block)
96
+ def self.before(&block)
98
97
  @before ||= []
99
98
  @before << block if block_given?
100
99
  @before
@@ -118,7 +117,7 @@ module Typhoeus
118
117
  # @return [ Object ] Returns the return value of the block.
119
118
  #
120
119
  # @see Typhoeus::Config#block_connection
121
- def with_connection
120
+ def self.with_connection
122
121
  old = Config.block_connection
123
122
  Config.block_connection = false
124
123
  result = yield if block_given?
@@ -85,6 +85,22 @@ module Typhoeus
85
85
  #
86
86
  # @return [ Ethon::Easy ] The easy.
87
87
  def set_callback
88
+ if request.streaming?
89
+ response = nil
90
+ easy.on_headers do |easy|
91
+ response = Response.new(Ethon::Easy::Mirror.from_easy(easy).options)
92
+ request.execute_headers_callbacks(response)
93
+ end
94
+ easy.on_body do |chunk, easy|
95
+ request.on_body.each do |callback|
96
+ callback.call(chunk, response)
97
+ end
98
+ end
99
+ else
100
+ easy.on_headers do |easy|
101
+ request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options))
102
+ end
103
+ end
88
104
  easy.on_complete do |easy|
89
105
  request.finish(Response.new(easy.mirror.options))
90
106
  Typhoeus::Pool.release(easy)
@@ -3,6 +3,7 @@ module Typhoeus
3
3
  module Cacheable
4
4
  def add(request)
5
5
  if request.cacheable? && response = Typhoeus::Config.cache.get(request)
6
+ response.cached = true
6
7
  request.finish(response)
7
8
  dequeue
8
9
  else
data/lib/typhoeus/pool.rb CHANGED
@@ -4,20 +4,18 @@ module Typhoeus
4
4
 
5
5
  # The easy pool stores already initialized
6
6
  # easy handles for future use. This is useful
7
- # because creating them is quite expensive.
7
+ # because creating them is expensive.
8
8
  #
9
9
  # @api private
10
10
  module Pool
11
- extend self
12
-
13
11
  @mutex = Mutex.new
14
12
 
15
13
  # Releases easy into the pool. The easy handle is
16
14
  # reset before it gets back in.
17
15
  #
18
16
  # @example Release easy.
19
- # hydra.release_easy(easy)
20
- def release(easy)
17
+ # Typhoeus::Pool.release(easy)
18
+ def self.release(easy)
21
19
  easy.reset
22
20
  @mutex.synchronize { easies << easy }
23
21
  end
@@ -25,18 +23,25 @@ module Typhoeus
25
23
  # Return an easy from the pool.
26
24
  #
27
25
  # @example Return easy.
28
- # hydra.get_easy
26
+ # Typhoeus::Pool.get
29
27
  #
30
28
  # @return [ Ethon::Easy ] The easy.
31
- def get
29
+ def self.get
32
30
  @mutex.synchronize { easies.pop } || Ethon::Easy.new
33
31
  end
34
32
 
35
- def clear
33
+ # Clear the pool
34
+ def self.clear
36
35
  @mutex.synchronize { easies.clear }
37
36
  end
38
37
 
39
- def with_easy(&block)
38
+ # Use yielded easy, will be released automatically afterwards.
39
+ #
40
+ # @example Use easy.
41
+ # Typhoeus::Pool.with_easy do |easy|
42
+ # # use easy
43
+ # end
44
+ def self.with_easy(&block)
40
45
  easy = get
41
46
  yield easy
42
47
  ensure
@@ -45,13 +50,7 @@ module Typhoeus
45
50
 
46
51
  private
47
52
 
48
- # Return the easy pool.
49
- #
50
- # @example Return easy pool.
51
- # hydra.easy_pool
52
- #
53
- # @return [ Array<Ethon::Easy> ] The easy pool.
54
- def easies
53
+ def self.easies
55
54
  @easies ||= []
56
55
  end
57
56
  end
@@ -8,6 +8,7 @@ require 'typhoeus/request/marshal'
8
8
  require 'typhoeus/request/memoizable'
9
9
  require 'typhoeus/request/operations'
10
10
  require 'typhoeus/request/responseable'
11
+ require 'typhoeus/request/streamable'
11
12
  require 'typhoeus/request/stubbable'
12
13
 
13
14
  module Typhoeus
@@ -24,6 +25,7 @@ module Typhoeus
24
25
  extend Request::Actions
25
26
  include Request::Callbacks::Types
26
27
  include Request::Callbacks
28
+ include Request::Streamable
27
29
  include Request::Marshal
28
30
  include Request::Operations
29
31
  include Request::Responseable
@@ -193,6 +195,7 @@ module Typhoeus
193
195
  @options[:headers] = {'User-Agent' => Typhoeus::USER_AGENT}
194
196
  end
195
197
  @options[:verbose] = Typhoeus::Config.verbose if @options[:verbose].nil? && !Typhoeus::Config.verbose.nil?
198
+ @options[:maxredirs] ||= 50
196
199
  end
197
200
  end
198
201
  end
@@ -2,7 +2,7 @@ module Typhoeus
2
2
  class Request
3
3
  module Cacheable
4
4
  def response=(response)
5
- Typhoeus::Config.cache.set(self, response) if cacheable?
5
+ Typhoeus::Config.cache.set(self, response) if cacheable? && !response.cached?
6
6
  super
7
7
  end
8
8
 
@@ -12,6 +12,7 @@ module Typhoeus
12
12
 
13
13
  def run
14
14
  if cacheable? && response = Typhoeus::Config.cache.get(self)
15
+ response.cached = true
15
16
  finish(response)
16
17
  else
17
18
  super
@@ -74,9 +74,39 @@ module Typhoeus
74
74
  @on_failure << block if block_given?
75
75
  @on_failure
76
76
  end
77
+
78
+ # Set on_headers callback.
79
+ #
80
+ # @example Set on_headers.
81
+ # request.on_headers { |response| p "yay" }
82
+ #
83
+ # @param [ Block ] block The block to execute.
84
+ #
85
+ # @yield [ Typhoeus::Response ]
86
+ #
87
+ # @return [ Array<Block> ] All on_headers blocks.
88
+ def on_headers(&block)
89
+ @on_headers ||= []
90
+ @on_headers << block if block_given?
91
+ @on_headers
92
+ end
93
+ end
94
+
95
+ # Execute the headers callbacks and yields response.
96
+ #
97
+ # @example Execute callbacks.
98
+ # request.execute_headers_callbacks
99
+ #
100
+ # @return [ Array<Object> ] The results of the on_headers callbacks.
101
+ #
102
+ # @api private
103
+ def execute_headers_callbacks(response)
104
+ (Typhoeus.on_headers + on_headers).map do |callback|
105
+ callback.call(response)
106
+ end
77
107
  end
78
108
 
79
- # Execute nessecary callback and yields response. This
109
+ # Execute necessary callback and yields response. This
80
110
  # include in every case on_complete, on_success if
81
111
  # successful and on_failure if not.
82
112
  #
@@ -7,7 +7,7 @@ module Typhoeus
7
7
  # Return the important data needed to serialize this Request, except the
8
8
  # `on_complete`, `on_success`, `on_failure`, and `hydra`, since they cannot be marshalled.
9
9
  def marshal_dump
10
- unmarshallable = %w(@on_complete @on_success @on_failure @hydra)
10
+ unmarshallable = %w(@on_complete @on_success @on_failure @on_headers @on_body @hydra)
11
11
  (instance_variables - unmarshallable - unmarshallable.map(&:to_sym)).map do |name|
12
12
  [name, instance_variable_get(name)]
13
13
  end
@@ -0,0 +1,34 @@
1
+ module Typhoeus
2
+ class Request
3
+
4
+ # This module contians the logic for response streaming.
5
+ module Streamable
6
+
7
+ # Set on_body callback.
8
+ #
9
+ # This callback will be called each time a portion of the body is read from the socket.
10
+ # Setting an on_body callback will cause the response body to be empty.
11
+ #
12
+ # @example Set on_body.
13
+ # request.on_body { |response, body_chunk| puts "Got #{body_chunk.bytesize} bytes" }
14
+ #
15
+ # @param [ Block ] block The block to execute.
16
+ #
17
+ # @yield [ Typhoeus::Response, String ]
18
+ #
19
+ # @return [ Array<Block> ] All on_body blocks.
20
+ def on_body(&block)
21
+ @on_body ||= []
22
+ @on_body << block if block_given?
23
+ @on_body
24
+ end
25
+
26
+ # Is this request using streaming?
27
+ #
28
+ # @return [ Boolean ] True if any on_body blocks have been set.
29
+ def streaming?
30
+ @on_body && @on_body.any?
31
+ end
32
+ end
33
+ end
34
+ end
@@ -1,6 +1,7 @@
1
1
  require 'typhoeus/response/header'
2
2
  require 'typhoeus/response/informations'
3
3
  require 'typhoeus/response/status'
4
+ require 'typhoeus/response/cacheable'
4
5
 
5
6
  module Typhoeus
6
7
 
@@ -8,6 +9,7 @@ module Typhoeus
8
9
  class Response
9
10
  include Response::Informations
10
11
  include Response::Status
12
+ include Response::Cacheable
11
13
 
12
14
  # Remembers the corresponding request.
13
15
  #
@@ -0,0 +1,14 @@
1
+ module Typhoeus
2
+ class Response
3
+ module Cacheable
4
+
5
+ # Set the cache status, if we got response from cache
6
+ # it will have cached? == true
7
+ attr_writer :cached
8
+
9
+ def cached?
10
+ !!@cached
11
+ end
12
+ end
13
+ end
14
+ end
@@ -1,5 +1,5 @@
1
1
  module Typhoeus
2
2
 
3
3
  # The current Typhoeus version.
4
- VERSION = '0.6.6'
4
+ VERSION = '0.6.7'
5
5
  end
@@ -43,10 +43,16 @@ describe Typhoeus::Hydra::Cacheable do
43
43
  let(:response) { Typhoeus::Response.new }
44
44
  before { cache.memory[request] = response }
45
45
 
46
+ it "returns response with cached status" do
47
+ hydra.add(request)
48
+ expect(response.cached?).to be_true
49
+ end
50
+
46
51
  context "when no queued requests" do
47
52
  it "finishes request" do
48
53
  request.should_receive(:finish).with(response)
49
54
  hydra.add(request)
55
+ expect(response.cached?).to be_true
50
56
  end
51
57
  end
52
58
 
@@ -55,7 +61,7 @@ describe Typhoeus::Hydra::Cacheable do
55
61
 
56
62
  before { cache.memory[queued_request] = response }
57
63
 
58
- it "finishesh both requests" do
64
+ it "finishes both requests" do
59
65
  hydra.queue(queued_request)
60
66
  request.should_receive(:finish).with(response)
61
67
  queued_request.should_receive(:finish).with(response)
@@ -19,8 +19,4 @@ describe Typhoeus::Hydra do
19
19
  expect(Typhoeus::Hydra.hydra).to be_a(Typhoeus::Hydra)
20
20
  end
21
21
  end
22
-
23
- describe "#fire_and_forget" do
24
- it
25
- end
26
22
  end
@@ -32,6 +32,11 @@ describe Typhoeus::Request::Cacheable do
32
32
  request.response = response
33
33
  expect(cache.memory[request]).to be
34
34
  end
35
+
36
+ it "doesn't set cached on response" do
37
+ request.response = response
38
+ expect(request.response.cached?).to be_false
39
+ end
35
40
  end
36
41
 
37
42
  context "when request in memory" do
@@ -41,6 +46,11 @@ describe Typhoeus::Request::Cacheable do
41
46
  request.should_receive(:finish).with(response)
42
47
  request.run
43
48
  end
49
+
50
+ it "sets cached to true for response" do
51
+ request.run
52
+ expect(request.response.cached?).to be_true
53
+ end
44
54
  end
45
55
  end
46
56
  end
@@ -33,6 +33,37 @@ describe Typhoeus::Request::Operations do
33
33
  request.run
34
34
  end
35
35
 
36
+ it "calls on_body" do
37
+ on_body_called = false
38
+ request.on_body { |body, response| on_body_called = true }
39
+ request.run
40
+ expect(on_body_called).to be_true
41
+ expect(request.response.body).to satisfy { |v| v.nil? || v == '' }
42
+ end
43
+
44
+ it "makes response headers available to on_body" do
45
+ headers = nil
46
+ request.on_body { |body, response| headers = response.headers }
47
+ request.run
48
+ expect(headers).to be
49
+ expect(headers).to eq(request.response.headers)
50
+ end
51
+
52
+ it "calls on_headers and on_body" do
53
+ headers = nil
54
+ request.on_headers { |response| headers = response.headers }
55
+ request.on_body { |body, response| expect(headers).not_to be_nil ; expect(response.headers).to eq(headers) }
56
+ request.on_complete { |response| expect(response).not_to be_nil ; expect(response.headers).to eq(headers) ; expect(response.body).to be_empty }
57
+ request.run
58
+ end
59
+
60
+ it "calls on_headers and on_complete" do
61
+ headers = nil
62
+ request.on_headers { |response| headers = response.headers }
63
+ request.on_complete { |response| expect(response).not_to be_nil ; expect(response.headers).to eq(headers) ; expect(response.body).not_to be_empty }
64
+ request.run
65
+ end
66
+
36
67
  it "calls on_complete" do
37
68
  callback = double(:call)
38
69
  callback.should_receive(:call)
@@ -2,7 +2,7 @@ require 'spec_helper'
2
2
 
3
3
  describe Typhoeus::Request do
4
4
  let(:base_url) { "localhost:3001" }
5
- let(:options) { {:verbose => true, :headers => { 'User-Agent' => "Fubar" }} }
5
+ let(:options) { {:verbose => true, :headers => { 'User-Agent' => "Fubar" }, :maxredirs => 50} }
6
6
  let(:request) { Typhoeus::Request.new(base_url, options) }
7
7
 
8
8
  describe ".url" do
@@ -37,6 +37,17 @@ describe Typhoeus::Request do
37
37
  expect(request.options).to eq(options)
38
38
  end
39
39
 
40
+ it "stores original options" do
41
+ expect(request.original_options).to eq(options)
42
+ expect(request.original_options).to_not be(request.options)
43
+ end
44
+
45
+ it "sets defaults" do
46
+ expect(request.options[:headers]['User-Agent']).to be
47
+ end
48
+ end
49
+
50
+ describe "set_defaults" do
40
51
  context "when header with user agent" do
41
52
  let(:options) { {:headers => {'User-Agent' => "Custom"} } }
42
53
 
@@ -62,6 +73,22 @@ describe Typhoeus::Request do
62
73
  expect(request.options[:verbose]).to be_true
63
74
  end
64
75
  end
76
+
77
+ context "when maxredirs" do
78
+ context "when not set" do
79
+ it "defaults to 50" do
80
+ expect(request.options[:maxredirs]).to be(50)
81
+ end
82
+ end
83
+
84
+ context "when set" do
85
+ let(:options) { {:maxredirs => 1} }
86
+
87
+ it "respects" do
88
+ expect(request.options[:maxredirs]).to be(1)
89
+ end
90
+ end
91
+ end
65
92
  end
66
93
 
67
94
  describe "#eql?" do
@@ -78,4 +78,23 @@ describe Typhoeus::Response do
78
78
  end
79
79
  end
80
80
  end
81
+
82
+ describe "#cached" do
83
+ context "when @cached" do
84
+ before { response.cached = true }
85
+
86
+ it "returns cached status" do
87
+ expect(response.cached?).to be_true
88
+ end
89
+ end
90
+
91
+ context "when @cached is nil" do
92
+ before { response.cached = nil }
93
+
94
+ it "returns false" do
95
+ expect(response.cached?).to be_false
96
+ end
97
+ end
98
+
99
+ end
81
100
  end
data/typhoeus.gemspec CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.required_rubygems_version = ">= 1.3.6"
18
18
  s.license = 'MIT'
19
19
 
20
- s.add_dependency('ethon', ["~> 0.6.1"])
20
+ s.add_dependency('ethon', ["~> 0.6.2"])
21
21
 
22
22
  s.files = `git ls-files`.split("\n")
23
23
  s.test_files = `git ls-files -- spec/*`.split("\n")
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: typhoeus
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.6
4
+ version: 0.6.7
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -11,7 +11,7 @@ authors:
11
11
  autorequire:
12
12
  bindir: bin
13
13
  cert_chain: []
14
- date: 2013-10-31 00:00:00.000000000 Z
14
+ date: 2013-12-19 00:00:00.000000000 Z
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: ethon
@@ -20,7 +20,7 @@ dependencies:
20
20
  requirements:
21
21
  - - ~>
22
22
  - !ruby/object:Gem::Version
23
- version: 0.6.1
23
+ version: 0.6.2
24
24
  type: :runtime
25
25
  prerelease: false
26
26
  version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
28
28
  requirements:
29
29
  - - ~>
30
30
  - !ruby/object:Gem::Version
31
- version: 0.6.1
31
+ version: 0.6.2
32
32
  description: Like a modern code version of the mythical beast with 100 serpent heads,
33
33
  Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic.
34
34
  email:
@@ -80,8 +80,10 @@ files:
80
80
  - lib/typhoeus/request/memoizable.rb
81
81
  - lib/typhoeus/request/operations.rb
82
82
  - lib/typhoeus/request/responseable.rb
83
+ - lib/typhoeus/request/streamable.rb
83
84
  - lib/typhoeus/request/stubbable.rb
84
85
  - lib/typhoeus/response.rb
86
+ - lib/typhoeus/response/cacheable.rb
85
87
  - lib/typhoeus/response/header.rb
86
88
  - lib/typhoeus/response/informations.rb
87
89
  - lib/typhoeus/response/status.rb
@@ -140,7 +142,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
140
142
  version: '0'
141
143
  segments:
142
144
  - 0
143
- hash: -806955660153474198
145
+ hash: -1737493821567630899
144
146
  required_rubygems_version: !ruby/object:Gem::Requirement
145
147
  none: false
146
148
  requirements: