typhoeus 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: df7045af2adbad5bf0b5bc883e8bc259a2f63544
-  data.tar.gz: 16d738d8b00c774fe54a78c32becde597ad69569
+SHA256:
+  metadata.gz: 417ef8dd5903c12a896c84db243131ec364aec235a44ac32d2590d95f919c10a
+  data.tar.gz: 62fca0f2cf069282d5c6cdbed018e395b6b40f379cb2d8e45129a156e7aed69a
 SHA512:
-  metadata.gz: c5f8427b91d383cd94bc48c4f944b3bc7dbce619e0eccda0f84d428eb575ff9f7b65530f18789cb1aa99f02e4f03293af9987cffb12686542af13b9fb49bba51
-  data.tar.gz: e01422dc678ff40ac31b695bc1dea8d0c3b843e0ecd2ad61fa386d9b290a0950db93e37289215aedaee3aad60a7cc2acd48efc4baf59ae4fc96f3198d5df460a
+  metadata.gz: 9a8e291edbdb7e635f6ffbf085783752e6a0688e342dbec333e2fd85f6d4ac841a4bfca128fe36d4f30d5e0f87a8d124447d77cfd2d15e77c6f29bec4a55f197
+  data.tar.gz: cf6d163bfb31f1cf77c2faf068601b6c65685d5afaf0f582dadd9505882fb8c567583fd1dad7a47f27210018e992ac300a7803f3870250e1f7a1461111bcab80

data/README.md

@@ -191,6 +191,19 @@ end
 request.run
 ```
 
+If you need to interrupt the stream halfway,
+you can return the `:abort` symbol from the `on_body` block, example:
+
+```ruby
+request.on_body do |chunk|
+  buffer << chunk
+  :abort if buffer.size > 1024 * 1024
+end
+```
+
+This will properly stop the stream internally and avoid any memory leak which
+may happen if you interrupt with something like a `return`, `throw` or `raise`.
+
 ### Making Parallel Requests
 
 Generally, you should be running requests through hydra. Here is how that looks:
@@ -242,6 +255,33 @@ end
 hydra.run
 ```
 
+### Making Parallel Requests with Faraday + Typhoeus
+
+```ruby
+require 'faraday'
+
+conn = Faraday.new(:url => 'http://httppage.com') do |builder|
+  builder.request  :url_encoded
+  builder.response :logger
+  builder.adapter  :typhoeus
+end
+
+conn.in_parallel do
+  response1 = conn.get('/first')
+  response2 = conn.get('/second')
+
+  # these will return nil here since the
+  # requests have not been completed
+  response1.body
+  response2.body
+end
+
+# after it has been completed the response information is fully available
+# response1.status, etc
+response1.body
+response2.body
+```
+
 ### Specifying Max Concurrency
 
 Hydra will also handle how many requests you can make in parallel. Things will get flakey if you try to make too many requests at the same time. The built in limit is 200. When more requests than that are queued up, hydra will save them for later and start the requests as others are finished. You can raise or lower the concurrency limit through the Hydra constructor.
@@ -327,6 +367,14 @@ Typhoeus::Config.cache = Typhoeus::Cache::Redis.new(redis)
 All three of these adapters take an optional keyword argument `default_ttl`, which sets a default
 TTL on cached responses (in seconds), for requests which do not have a cache TTL set.
 
+You may also selectively choose not to cache by setting `cache` to `false` on a request or to use
+a different adapter.
+
+```ruby
+cache = Cache.new
+Typhoeus.get("www.example.com", cache: cache)
+```
+
 ### Direct Stubbing
 
 Hydra allows you to stub out specific urls and patterns to avoid hitting

data/lib/typhoeus/adapters/faraday.rb

@@ -99,7 +99,7 @@ module Faraday # :nodoc:
             unless parallel?(env)
               raise Faraday::Error::TimeoutError, "request timed out"
             end
-          elsif resp.response_code == 0
+          elsif (resp.response_code == 0) || ((resp.return_code != :ok) && !resp.mock?)
             env[:typhoeus_connection_failed] = true
             env[:typhoeus_return_message] = resp.return_message
             unless parallel?(env)

data/lib/typhoeus/easy_factory.rb

@@ -35,7 +35,7 @@ module Typhoeus
 
     REMOVED_OPTIONS =  Set.new([:cache_key_basis, :cache_timeout, :user_agent])
 
-    SANITIZE_IGNORE  = Set.new([:method, :cache_ttl])
+    SANITIZE_IGNORE  = Set.new([:method, :cache_ttl, :cache])
     SANITIZE_TIMEOUT = Set.new([:timeout_ms, :connecttimeout_ms])
 
     # Returns the request provided.
@@ -155,6 +155,11 @@ module Typhoeus
           request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options))
         end
       end
+      request.on_progress.each do |callback|
+        easy.on_progress do |dltotal, dlnow, ultotal, ulnow, easy|
+          callback.call(dltotal, dlnow, ultotal, ulnow, response)
+        end
+      end
       easy.on_complete do |easy|
         request.finish(Response.new(easy.mirror.options))
         Typhoeus::Pool.release(easy)

data/lib/typhoeus/request/actions.rb

@@ -15,7 +15,7 @@ module Typhoeus
       #
       # @option (see Typhoeus::Request#initialize)
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def get(base_url, options = {})
@@ -31,7 +31,7 @@ module Typhoeus
       #
       # @option (see Typhoeus::Request#initialize)
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def post(base_url, options = {})
@@ -50,7 +50,7 @@ module Typhoeus
       # @option options :body [ Hash ] Body hash which
       #   becomes a PUT request body.
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def put(base_url, options = {})
@@ -66,7 +66,7 @@ module Typhoeus
       #
       # @option (see Typhoeus::Request#initialize)
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def delete(base_url, options = {})
@@ -82,7 +82,7 @@ module Typhoeus
       #
       # @option (see Typhoeus::Request#initialize)
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def head(base_url, options = {})
@@ -98,7 +98,7 @@ module Typhoeus
       #
       # @option (see Typhoeus::Request#initialize)
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def patch(base_url, options = {})
@@ -114,7 +114,7 @@ module Typhoeus
       #
       # @option (see Typhoeus::Request#initialize)
       #
-      # @return (see Typhoeus::Request#initialize)
+      # @return (see Typhoeus::Response#initialize)
       #
       # @note (see Typhoeus::Request#initialize)
       def options(base_url, options = {})

data/lib/typhoeus/request/cacheable.rb

@@ -2,16 +2,16 @@ module Typhoeus
   class Request
     module Cacheable
       def response=(response)
-        Typhoeus::Config.cache.set(self, response) if cacheable? && !response.cached?
+        cache.set(self, response) if cacheable? && !response.cached?
         super
       end
 
       def cacheable?
-        Typhoeus::Config.cache
+        cache
       end
 
       def run
-        if cacheable? && response = Typhoeus::Config.cache.get(self)
+        if cacheable? && response = cache.get(self)
           response.cached = true
           finish(response)
         else
@@ -22,6 +22,13 @@ module Typhoeus
       def cache_ttl
         options[:cache_ttl]
       end
+
+      private
+
+      def cache
+        return nil if options[:cache] === false
+        options[:cache] || Typhoeus::Config.cache
+      end
     end
   end
 end

data/lib/typhoeus/request/callbacks.rb

@@ -89,6 +89,24 @@ module Typhoeus
           @on_headers << block if block_given?
           @on_headers
         end
+
+        # Set on_progress callback.
+        #
+        # @example Set on_progress.
+        #   request.on_progress do |dltotal, dlnow, ultotal, ulnow|
+        #     puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})"
+        #   end
+        #
+        # @param [ Block ] block The block to execute.
+        #
+        # @yield [ Typhoeus::Response ]
+        #
+        # @return [ Array<Block> ] All on_progress blocks.
+        def on_progress(&block)
+          @on_progress ||= []
+          @on_progress << block if block_given?
+          @on_progress
+        end
       end
 
       # Execute the headers callbacks and yields response.
@@ -106,8 +124,8 @@ module Typhoeus
       end
 
       # Execute necessary callback and yields response. This
-      # include in every case on_complete, on_success if
-      # successful and on_failure if not.
+      # include in every case on_complete and on_progress, on_success
+      # if successful and on_failure if not.
       #
       # @example Execute callbacks.
       #   request.execute_callbacks
@@ -116,7 +134,7 @@ module Typhoeus
       #
       # @api private
       def execute_callbacks
-        callbacks = Typhoeus.on_complete + on_complete
+        callbacks = Typhoeus.on_complete + Typhoeus.on_progress + on_complete + on_progress
 
         if response && response.success?
           callbacks += Typhoeus.on_success + on_success

data/lib/typhoeus/request/marshal.rb

@@ -5,9 +5,9 @@ module Typhoeus
     module Marshal
 
       # Return the important data needed to serialize this Request, except the
-      # `on_complete`, `on_success`, `on_failure`, and `hydra`, since they cannot be marshalled.
+      # request callbacks and `hydra`, since they cannot be marshalled.
       def marshal_dump
-        unmarshallable = %w(@on_complete @on_success @on_failure @on_headers @on_body @hydra)
+        unmarshallable = %w(@on_complete @on_success @on_failure @on_progress @on_headers @on_body @hydra)
         (instance_variables - unmarshallable - unmarshallable.map(&:to_sym)).map do |name|
           [name, instance_variable_get(name)]
         end

data/lib/typhoeus/request/streamable.rb

@@ -10,7 +10,7 @@ module Typhoeus
       # Setting an on_body callback will cause the response body to be empty.
       #
       # @example Set on_body.
-      #   request.on_body { |response, body_chunk| puts "Got #{body_chunk.bytesize} bytes" }
+      #   request.on_body { |body_chunk, response| puts "Got #{body_chunk.bytesize} bytes" }
       #
       # @param [ Block ] block The block to execute.
       #

data/lib/typhoeus/response.rb

@@ -14,7 +14,7 @@ module Typhoeus
     # Remembers the corresponding request.
     #
     # @example Get request.
-    #   request = Typhoeus::Request.get("www.example.com")
+    #   request = Typhoeus::Request.new("www.example.com")
     #   response = request.run
     #   request == response.request
     #   #=> true

data/lib/typhoeus/response/header.rb

@@ -1,3 +1,5 @@
+require 'delegate'
+
 module Typhoeus
   class Response
 

data/lib/typhoeus/response/status.rb

@@ -39,7 +39,7 @@ module Typhoeus
         @http_version ||= first_header_line ? first_header_line[/HTTP\/(\S+)/, 1] : nil
       end
 
-      # Return wether the response is a success.
+      # Return whether the response is a success.
       #
       # @example Return if the response was successful.
       #  response.success?
@@ -49,13 +49,12 @@ module Typhoeus
         (mock || return_code == :ok) && response_code && has_good_response_code?
       end
 
-      # Return wether the response is a failure.
+      # Return whether the response is a failure.
       #
       # @example Return if the response was failed.
       #  response.failure?
       #
       # @return [ Boolean ] Return true if failure, false else.
-
       def failure?
         (mock || return_code == :internal_server_error) && response_code && has_bad_response_code?
       end

data/lib/typhoeus/version.rb

@@ -1,5 +1,5 @@
 module Typhoeus
 
   # The current Typhoeus version.
-  VERSION = '1.3.0'
+  VERSION = '1.3.1'
 end

data/spec/typhoeus/easy_factory_spec.rb

@@ -104,6 +104,12 @@ describe Typhoeus::EasyFactory do
   end
 
   describe "#set_callback" do
+    it "sets easy.on_progress callback when an on_progress callback is provided" do
+      request.on_progress { 1 }
+      expect(easy_factory.easy).to receive(:on_progress)
+      easy_factory.send(:set_callback)
+    end
+
     it "sets easy.on_complete callback" do
       expect(easy_factory.easy).to receive(:on_complete)
       easy_factory.send(:set_callback)

data/spec/typhoeus/request/cacheable_spec.rb

@@ -55,6 +55,30 @@ describe Typhoeus::Request::Cacheable do
           request.run
         end
       end
+
+      context "when cache is specified on a request" do
+        before { Typhoeus::Config.cache = false }
+
+        context "when cache is false" do
+          let(:options) { { :cache => false } }
+
+          it "finishes request" do
+            expect(request.response).to_not be(response)
+            request.run
+          end
+        end
+
+        context "when cache is defined" do
+          let(:options) { { :cache => cache } }
+
+          before { cache.memory[request] = response }
+
+          it "finishes request" do
+            expect(request).to receive(:finish).with(response)
+            request.run
+          end
+        end
+      end
     end
   end
 

data/spec/typhoeus/request/callbacks_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Typhoeus::Request::Callbacks do
   let(:request) { Typhoeus::Request.new("fubar") }
 
-  [:on_complete, :on_success, :on_failure].each do |callback|
+  [:on_complete, :on_success, :on_failure, :on_progress].each do |callback|
     describe "##{callback}" do
       it "responds" do
         expect(request).to respond_to(callback)
@@ -33,7 +33,7 @@ describe Typhoeus::Request::Callbacks do
   end
 
   describe "#execute_callbacks" do
-    [:on_complete, :on_success, :on_failure].each do |callback|
+    [:on_complete, :on_success, :on_failure, :on_progress].each do |callback|
       context "when #{callback}" do
         context "when local callback" do
           before do

data/spec/typhoeus/request/marshal_spec.rb

@@ -5,7 +5,7 @@ describe Typhoeus::Request::Marshal do
   let(:request) { Typhoeus::Request.new(base_url) }
 
   describe "#marshal_dump" do
-    %w(on_complete on_success on_failure).each do |name|
+    %w(on_complete on_success on_failure on_progress).each do |name|
       context "when #{name} handler" do
         before { request.instance_variable_set("@#{name}", Proc.new{}) }
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: typhoeus
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - David Balatero
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-21 00:00:00.000000000 Z
+date: 2018-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ethon
@@ -150,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Parallel HTTP library on top of libcurl multi.