typhoeus 0.6.6 → 0.6.7

data/.travis.yml

@@ -12,6 +12,9 @@ rvm:
   - rbx-18mode
   - rbx-19mode
 matrix: 
+  fast_finish: true
   allow_failures:
     - rvm: ruby-head
     - rvm: jruby-head
+    - rvm: rbx-18mode
+    - rvm: rbx-19mode

data/CHANGELOG.md

@@ -4,6 +4,11 @@
 
 [Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.6...master)
 
+Enhancements:
+
+* Add response streaming.
+  ([\#339](https://github.com/typhoeus/typhoeus/pull/339))
+
 ## 0.6.6
 
 [Full Changelog](http://github.com/typhoeus/typhoeus/compare/v0.6.5...v0.6.6)

data/README.md

@@ -138,6 +138,31 @@ Typhoeus.post(
 )
 ```
 
+### Streaming the response body
+
+Typhoeus can stream responses. When you're expecting a large response,
+set the `on_body` callback on a request. Typhoeus will yield to the callback
+with chunks of the response, as they're read. When you set an `on_body` callback,
+Typhoeus will not store the complete response.
+
+```ruby
+downloaded_file = File.open 'huge.iso', 'wb'
+request = Typhoeus::Request.new("www.example.com/huge.iso")
+request.on_headers do |response|
+  if ! response.success?
+    raise "Request failed"
+  end
+end
+request.on_body do |chunk|
+  downloaded_file.write(chunk)
+end
+request.on_complete do |response|
+  downloaded_file.close
+  # Note that response.body is ""
+end
+request.run
+```
+
 ### Making Parallel Requests
 
 Generally, you should be running requests through hydra. Here is how that looks
@@ -211,8 +236,8 @@ class Cache
     memory[request]
   end
 
-  def set(request, reseponse)
-    memory[request] = reseponse
+  def set(request, response)
+    memory[request] = response
   end
 end
 
@@ -245,6 +270,16 @@ Typhoeus.get("www.example.com") == response
 #=> true
 ```
 
+When testing make sure to clear your expectations or the stubs will persist between tests. The following can be included in your spec_helper.rb file to do this automatically.
+
+```ruby
+RSpec.configure do |config|
+  config.before :each do
+    Typhoeus::Expectation.clear
+  end
+end
+```
+
 ### Timeouts
 
 No exceptions are raised on HTTP timeouts. You can check whether a request timed out with the following methods:
@@ -368,3 +403,7 @@ THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 OTHER DEALINGS IN THE SOFTWARE.
+
+
+[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/typhoeus/typhoeus/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
+

data/lib/typhoeus.rb

@@ -43,7 +43,6 @@ end
 #
 # @since 0.5.0
 module Typhoeus
-  extend self
   extend Request::Actions
   extend Request::Callbacks::Types
 
@@ -59,7 +58,7 @@ module Typhoeus
   # @return [ Typhoeus::Config ] The configuration.
   #
   # @see Typhoeus::Config
-  def configure
+  def self.configure
     yield Config
   end
 
@@ -73,7 +72,7 @@ module Typhoeus
   # @return [ Typhoeus::Expectation ] The expecatation.
   #
   # @see Typhoeus::Expectation
-  def stub(base_url, options = {}, &block)
+  def self.stub(base_url, options = {}, &block)
     expectation = Expectation.all.find{ |e| e.base_url == base_url && e.options == options }
     if expectation.nil?
       expectation = Expectation.new(base_url, options)
@@ -94,7 +93,7 @@ module Typhoeus
   # @yield [ Typhoeus::Request ]
   #
   # @return [ Array<Block> ] All before blocks.
-  def before(&block)
+  def self.before(&block)
     @before ||= []
     @before << block if block_given?
     @before
@@ -118,7 +117,7 @@ module Typhoeus
   # @return [ Object ] Returns the return value of the block.
   #
   # @see Typhoeus::Config#block_connection
-  def with_connection
+  def self.with_connection
     old = Config.block_connection
     Config.block_connection = false
     result = yield if block_given?

data/lib/typhoeus/easy_factory.rb

@@ -85,6 +85,22 @@ module Typhoeus
     #
     # @return [ Ethon::Easy ] The easy.
     def set_callback
+      if request.streaming?
+        response = nil
+        easy.on_headers do |easy|
+          response = Response.new(Ethon::Easy::Mirror.from_easy(easy).options)
+          request.execute_headers_callbacks(response)
+        end
+        easy.on_body do |chunk, easy|
+          request.on_body.each do |callback|
+            callback.call(chunk, response)
+          end
+        end
+      else
+        easy.on_headers do |easy|
+          request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options))
+        end
+      end
       easy.on_complete do |easy|
         request.finish(Response.new(easy.mirror.options))
         Typhoeus::Pool.release(easy)

data/lib/typhoeus/hydra/cacheable.rb

@@ -3,6 +3,7 @@ module Typhoeus
     module Cacheable
       def add(request)
         if request.cacheable? && response = Typhoeus::Config.cache.get(request)
+          response.cached = true
           request.finish(response)
           dequeue
         else

data/lib/typhoeus/pool.rb

@@ -4,20 +4,18 @@ module Typhoeus
 
   # The easy pool stores already initialized
   # easy handles for future use. This is useful
-  # because creating them is quite expensive.
+  # because creating them is expensive.
   #
   # @api private
   module Pool
-    extend self
-
     @mutex = Mutex.new
 
     # Releases easy into the pool. The easy handle is
     # reset before it gets back in.
     #
     # @example Release easy.
-    #   hydra.release_easy(easy)
-    def release(easy)
+    #   Typhoeus::Pool.release(easy)
+    def self.release(easy)
       easy.reset
       @mutex.synchronize { easies << easy }
     end
@@ -25,18 +23,25 @@ module Typhoeus
     # Return an easy from the pool.
     #
     # @example Return easy.
-    #   hydra.get_easy
+    #   Typhoeus::Pool.get
     #
     # @return [ Ethon::Easy ] The easy.
-    def get
+    def self.get
       @mutex.synchronize { easies.pop } || Ethon::Easy.new
     end
 
-    def clear
+    # Clear the pool
+    def self.clear
       @mutex.synchronize { easies.clear }
     end
 
-    def with_easy(&block)
+    # Use yielded easy, will be released automatically afterwards.
+    #
+    # @example Use easy.
+    #   Typhoeus::Pool.with_easy do |easy|
+    #     # use easy
+    #   end
+    def self.with_easy(&block)
       easy = get
       yield easy
     ensure
@@ -45,13 +50,7 @@ module Typhoeus
 
     private
 
-    # Return the easy pool.
-    #
-    # @example Return easy pool.
-    #   hydra.easy_pool
-    #
-    # @return [ Array<Ethon::Easy> ] The easy pool.
-    def easies
+    def self.easies
       @easies ||= []
     end
   end

data/lib/typhoeus/request.rb

@@ -8,6 +8,7 @@ require 'typhoeus/request/marshal'
 require 'typhoeus/request/memoizable'
 require 'typhoeus/request/operations'
 require 'typhoeus/request/responseable'
+require 'typhoeus/request/streamable'
 require 'typhoeus/request/stubbable'
 
 module Typhoeus
@@ -24,6 +25,7 @@ module Typhoeus
     extend  Request::Actions
     include Request::Callbacks::Types
     include Request::Callbacks
+    include Request::Streamable
     include Request::Marshal
     include Request::Operations
     include Request::Responseable
@@ -193,6 +195,7 @@ module Typhoeus
         @options[:headers] = {'User-Agent' => Typhoeus::USER_AGENT}
       end
       @options[:verbose] = Typhoeus::Config.verbose if @options[:verbose].nil? && !Typhoeus::Config.verbose.nil?
+      @options[:maxredirs] ||= 50
     end
   end
 end

data/lib/typhoeus/request/cacheable.rb

@@ -2,7 +2,7 @@ module Typhoeus
   class Request
     module Cacheable
       def response=(response)
-        Typhoeus::Config.cache.set(self, response) if cacheable?
+        Typhoeus::Config.cache.set(self, response) if cacheable? && !response.cached?
         super
       end
 
@@ -12,6 +12,7 @@ module Typhoeus
 
       def run
         if cacheable? && response = Typhoeus::Config.cache.get(self)
+          response.cached = true
           finish(response)
         else
           super

data/lib/typhoeus/request/callbacks.rb

@@ -74,9 +74,39 @@ module Typhoeus
           @on_failure << block if block_given?
           @on_failure
         end
+
+        # Set on_headers callback.
+        #
+        # @example Set on_headers.
+        #   request.on_headers { |response| p "yay" }
+        #
+        # @param [ Block ] block The block to execute.
+        #
+        # @yield [ Typhoeus::Response ]
+        #
+        # @return [ Array<Block> ] All on_headers blocks.
+        def on_headers(&block)
+          @on_headers ||= []
+          @on_headers << block if block_given?
+          @on_headers
+        end
+      end
+
+      # Execute the headers callbacks and yields response.
+      #
+      # @example Execute callbacks.
+      #   request.execute_headers_callbacks
+      #
+      # @return [ Array<Object> ] The results of the on_headers callbacks.
+      #
+      # @api private
+      def execute_headers_callbacks(response)
+        (Typhoeus.on_headers + on_headers).map do |callback|
+          callback.call(response)
+        end
       end
 
-      # Execute nessecary callback and yields response. This
+      # Execute necessary callback and yields response. This
       # include in every case on_complete, on_success if
       # successful and on_failure if not.
       #

data/lib/typhoeus/request/marshal.rb

@@ -7,7 +7,7 @@ module Typhoeus
       # Return the important data needed to serialize this Request, except the
       # `on_complete`, `on_success`, `on_failure`, and `hydra`, since they cannot be marshalled.
       def marshal_dump
-        unmarshallable = %w(@on_complete @on_success @on_failure @hydra)
+        unmarshallable = %w(@on_complete @on_success @on_failure @on_headers @on_body @hydra)
         (instance_variables - unmarshallable - unmarshallable.map(&:to_sym)).map do |name|
           [name, instance_variable_get(name)]
         end

data/lib/typhoeus/request/streamable.rb

@@ -0,0 +1,34 @@
+module Typhoeus
+  class Request
+
+    # This module contians the logic for response streaming.
+    module Streamable
+
+      # Set on_body callback.
+      #
+      # This callback will be called each time a portion of the body is read from the socket.
+      # Setting an on_body callback will cause the response body to be empty.
+      #
+      # @example Set on_body.
+      #   request.on_body { |response, body_chunk| puts "Got #{body_chunk.bytesize} bytes" }
+      #
+      # @param [ Block ] block The block to execute.
+      #
+      # @yield [ Typhoeus::Response, String ]
+      #
+      # @return [ Array<Block> ] All on_body blocks.
+      def on_body(&block)
+        @on_body ||= []
+        @on_body << block if block_given?
+        @on_body
+      end
+
+      # Is this request using streaming?
+      #
+      # @return [ Boolean ] True if any on_body blocks have been set.
+      def streaming?
+        @on_body && @on_body.any?
+      end
+    end
+  end
+end

data/lib/typhoeus/response.rb

@@ -1,6 +1,7 @@
 require 'typhoeus/response/header'
 require 'typhoeus/response/informations'
 require 'typhoeus/response/status'
+require 'typhoeus/response/cacheable'
 
 module Typhoeus
 
@@ -8,6 +9,7 @@ module Typhoeus
   class Response
     include Response::Informations
     include Response::Status
+    include Response::Cacheable
 
     # Remembers the corresponding request.
     #

data/lib/typhoeus/response/cacheable.rb

@@ -0,0 +1,14 @@
+module Typhoeus
+  class Response
+    module Cacheable
+
+      # Set the cache status, if we got response from cache
+      # it will have cached? == true
+      attr_writer :cached
+
+      def cached?
+        !!@cached
+      end
+    end
+  end
+end

data/lib/typhoeus/version.rb

@@ -1,5 +1,5 @@
 module Typhoeus
 
   # The current Typhoeus version.
-  VERSION = '0.6.6'
+  VERSION = '0.6.7'
 end

data/spec/typhoeus/hydra/cacheable_spec.rb

@@ -43,10 +43,16 @@ describe Typhoeus::Hydra::Cacheable do
         let(:response) { Typhoeus::Response.new }
         before { cache.memory[request] = response }
 
+        it "returns response with cached status" do
+          hydra.add(request)
+          expect(response.cached?).to be_true
+        end
+
         context "when no queued requests" do
           it "finishes request" do
             request.should_receive(:finish).with(response)
             hydra.add(request)
+            expect(response.cached?).to be_true
           end
         end
 
@@ -55,7 +61,7 @@ describe Typhoeus::Hydra::Cacheable do
 
           before { cache.memory[queued_request] = response }
 
-          it "finishesh both requests" do
+          it "finishes both requests" do
             hydra.queue(queued_request)
             request.should_receive(:finish).with(response)
             queued_request.should_receive(:finish).with(response)

data/spec/typhoeus/hydra_spec.rb

@@ -19,8 +19,4 @@ describe Typhoeus::Hydra do
       expect(Typhoeus::Hydra.hydra).to be_a(Typhoeus::Hydra)
     end
   end
-
-  describe "#fire_and_forget" do
-    it
-  end
 end

data/spec/typhoeus/request/cacheable_spec.rb

@@ -32,6 +32,11 @@ describe Typhoeus::Request::Cacheable do
           request.response = response
           expect(cache.memory[request]).to be
         end
+
+        it "doesn't set cached on response" do
+          request.response = response
+          expect(request.response.cached?).to be_false
+        end
       end
 
       context "when request in memory" do
@@ -41,6 +46,11 @@ describe Typhoeus::Request::Cacheable do
           request.should_receive(:finish).with(response)
           request.run
         end
+
+        it "sets cached to true for response" do
+          request.run
+          expect(request.response.cached?).to be_true
+        end
       end
     end
   end

data/spec/typhoeus/request/operations_spec.rb

@@ -33,6 +33,37 @@ describe Typhoeus::Request::Operations do
       request.run
     end
 
+    it "calls on_body" do
+      on_body_called = false
+      request.on_body { |body, response| on_body_called = true }
+      request.run
+      expect(on_body_called).to be_true
+      expect(request.response.body).to satisfy { |v| v.nil? || v == '' }
+    end
+
+    it "makes response headers available to on_body" do
+      headers = nil
+      request.on_body { |body, response| headers = response.headers }
+      request.run
+      expect(headers).to be
+      expect(headers).to eq(request.response.headers)
+    end
+
+    it "calls on_headers and on_body" do
+      headers = nil
+      request.on_headers { |response| headers = response.headers }
+      request.on_body { |body, response| expect(headers).not_to be_nil ; expect(response.headers).to eq(headers) }
+      request.on_complete { |response| expect(response).not_to be_nil ; expect(response.headers).to eq(headers) ; expect(response.body).to be_empty }
+      request.run
+    end
+
+    it "calls on_headers and on_complete" do
+      headers = nil
+      request.on_headers { |response| headers = response.headers }
+      request.on_complete { |response| expect(response).not_to be_nil ; expect(response.headers).to eq(headers) ; expect(response.body).not_to be_empty }
+      request.run
+    end
+
     it "calls on_complete" do
       callback = double(:call)
       callback.should_receive(:call)

data/spec/typhoeus/request_spec.rb

@@ -2,7 +2,7 @@ require 'spec_helper'
 
 describe Typhoeus::Request do
   let(:base_url) { "localhost:3001" }
-  let(:options) { {:verbose => true, :headers => { 'User-Agent' => "Fubar" }} }
+  let(:options) { {:verbose => true, :headers => { 'User-Agent' => "Fubar" }, :maxredirs => 50} }
   let(:request) { Typhoeus::Request.new(base_url, options) }
 
   describe ".url" do
@@ -37,6 +37,17 @@ describe Typhoeus::Request do
       expect(request.options).to eq(options)
     end
 
+    it "stores original options" do
+      expect(request.original_options).to eq(options)
+      expect(request.original_options).to_not be(request.options)
+    end
+
+    it "sets defaults" do
+      expect(request.options[:headers]['User-Agent']).to be
+    end
+  end
+
+  describe "set_defaults" do
     context "when header with user agent" do
       let(:options) { {:headers => {'User-Agent' => "Custom"} } }
 
@@ -62,6 +73,22 @@ describe Typhoeus::Request do
         expect(request.options[:verbose]).to be_true
       end
     end
+
+    context "when maxredirs" do
+      context "when not set" do
+        it "defaults to 50" do
+          expect(request.options[:maxredirs]).to be(50)
+        end
+      end
+
+      context "when set" do
+        let(:options) { {:maxredirs => 1} }
+
+        it "respects" do
+          expect(request.options[:maxredirs]).to be(1)
+        end
+      end
+    end
   end
 
   describe "#eql?" do

data/spec/typhoeus/response_spec.rb

@@ -78,4 +78,23 @@ describe Typhoeus::Response do
       end
     end
   end
+
+  describe "#cached" do
+    context "when @cached" do
+      before { response.cached = true }
+
+      it "returns cached status" do
+        expect(response.cached?).to be_true
+      end
+    end
+
+    context "when @cached is nil" do
+      before { response.cached = nil }
+
+      it "returns false" do
+        expect(response.cached?).to be_false
+      end
+    end
+
+  end
 end

data/typhoeus.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = ">= 1.3.6"
   s.license = 'MIT'
 
-  s.add_dependency('ethon', ["~> 0.6.1"])
+  s.add_dependency('ethon', ["~> 0.6.2"])
 
   s.files        = `git ls-files`.split("\n")
   s.test_files   = `git ls-files -- spec/*`.split("\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: typhoeus
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.6.7
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-31 00:00:00.000000000 Z
+date: 2013-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ethon
@@ -20,7 +20,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.6.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.6.2
 description: Like a modern code version of the mythical beast with 100 serpent heads,
   Typhoeus runs HTTP requests in parallel while cleanly encapsulating handling logic.
 email:
@@ -80,8 +80,10 @@ files:
 - lib/typhoeus/request/memoizable.rb
 - lib/typhoeus/request/operations.rb
 - lib/typhoeus/request/responseable.rb
+- lib/typhoeus/request/streamable.rb
 - lib/typhoeus/request/stubbable.rb
 - lib/typhoeus/response.rb
+- lib/typhoeus/response/cacheable.rb
 - lib/typhoeus/response/header.rb
 - lib/typhoeus/response/informations.rb
 - lib/typhoeus/response/status.rb
@@ -140,7 +142,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -806955660153474198
+      hash: -1737493821567630899
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: