restify 1.13.0 → 1.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2b5dacc128de6cbfed604453544f51eaeffd16b9b6cb83f5353ecca66d24a8d
-  data.tar.gz: 5c3eba335362aea288f851327fa96262e5b464b9a747b092955241bb362e9c0d
+  metadata.gz: c29c95a9b1c0b18f2a73b51c99da7cdddf163e1cee33c43272810f5557b1375b
+  data.tar.gz: 95f956684154058cc6f25b7d4ffebde6706864b00c1369e91c8f8b5fcc50f0d9
 SHA512:
-  metadata.gz: 32569c3544a1ac6734b3c81c1e2b6a46ea7dba324c1f71f5e85bd1df93b8b73bb70114a7cec37dd8e38e561a821e674e8c437c78757d78df75e1e51b1a5a79f8
-  data.tar.gz: 6f03d156fcc20e0dfba3d7fda0a5c7851ff5d9342fbb9b50c4d90d1f43bf65e92f0ecabb714c4023ce4ca8cfb95b231abd7ea5d1e05339e839ea046b17f3d1fe
+  metadata.gz: f420cc4d0c4ebbdade1bc1a780c24218b5eea707bd8e420c113d155d58067936464dc3d4807717de8f515ed586784c9f13476b46c66f68c8d1181501937db3a2
+  data.tar.gz: 71da11cdf50d25abc968af02c6157c63f1aa5703c88f7ae28c3394aa028d9f038013d220283510532f7b540dffc69d0abad5fe4952c4e27df4910a4009475ae5

data/CHANGELOG.md

@@ -1,11 +1,11 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 
-
 ## Unreleased
 ---
 
@@ -18,135 +18,227 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Breaks
 
 
+## 1.15.2 - (2021-12-23)
+
+---
+
+### Fixes
+
+- ActiveSupport v7.0 issues with cache module
+
+
+## 1.15.1 - (2021-07-15)
+
+---
+
+### Fixes
+
+- Typhoeus internal exception when request timed out
+
+## 1.15.0 - (2021-07-09)
+
+---
+
+### New
+
+- Improve memory usage when running lots of requests with typhoeus adapter
+- Use hydra for synchronous requests
+- Increased thread stability of typhoeus adapter (new internal queuing mechanism)
+
+### Changes
+
+- Use Ruby 2.5 as baseline for testing and linting
+- Add Ruby 3.0 to automated testing
+- Changed timing behavior for multiple requests due to new internal queuing mechanism for the typhoeus adapter
+
+## 1.14.0 - (2020-12-15)
+
+---
+
+### New
+
+- Allow making requests with non-JSON bodies and custom content types (#42)
+
 ## 1.13.0 - (2020-06-12)
+
 ---
 
 ### New
-* typhoeus: Support setting per-request libcurl options on adapter
-* typhoeus: Enable short TCP keepalive probes by default (5s/5s)
 
+- typhoeus: Support setting per-request libcurl options on adapter
+- typhoeus: Enable short TCP keepalive probes by default (5s/5s)
 
 ## 1.12.0 - (2020-04-01)
+
 ---
 
 ### Added
-* Explicit exception class for HTTP status code 410 (Gone)
+
+- Explicit exception class for HTTP status code 410 (Gone)
 
 ### Changed
 
 ### Fixed
-* `GatewayError` exception classes introduced in v1.11.0 now properly inherit from `ServerError` (#30)
 
+- `GatewayError` exception classes introduced in v1.11.0 now properly inherit from `ServerError` (#30)
 
 ## 1.11.0 - (2019-07-11)
+
 ### Added
-* Explicit exception classes for HTTP status codes 500, 502, 503, 504
+
+- Explicit exception classes for HTTP status codes 500, 502, 503, 504
 
 ## 1.10.0 - 2018-12-11
+
 ### Changed
+
 - Raise more specific error on a few status codes (#17)
 - Complete promises with an empty list (but a list) of dependencies (#18)
 
 ## 1.9.0 - 2018-11-13
+
 ### Changed
+
 - Do not raise error on 3XX responses but return responses
 
 ## 1.8.0 - 2018-08-22
+
 ### Added
+
 - Add HEAD request method (#16)
 
 ## 1.7.0 - 2018-08-15
+
 ### Added
+
 - Introduce promise dependency timeouts (#15)
 
 ## 1.6.0 - 2018-08-09
+
 ### Changed
+
 - Specify headers on restify clients and individual requests (#14)
 
 ## 1.5.0 - 2018-07-31
+
 ### Added
+
 - Add MessagePack processor enabled by default
 
 ### Changed
+
 - Tune typhoeus adapter to be more race-condition resilent
 
 ## 1.4.4 - 2018-07-13
+
 ### Added
+
 - Add `#request` to `NetworkError` to ease debugging
 
 ### Changed
+
 - Fix race condition in typhoeus adapter
 
 ## 1.4.3 - 2017-11-15
+
 ### Added
+
 - Add advanced logging capabilities using logging gem
 
 ### Changed
+
 - Improve compatibility with webmocks returning `nil` as headers
 
 ## 1.4.1 - 2017-11-15
+
 ### Changed
+
 - Fix possible deadlock issues
 
 ## 1.4.0 - 2017-11-10
+
 ### Added
+
 - Add timeout option to requests (only supported by typhoeus adapter)
 
 ### Changed
+
 - Fix possible concurrency issue with typhoeus adapter
 
 ## 1.3.1 - 2017-11-10
+
 ### Changed
+
 - Improve typhoeus adapters initial request queuing
 - Disable default pipelining
 
 ## 1.3.0 - 2017-11-08
+
 ### Changed
+
 - Improve typhoeus adapter to better utilize concurrency
 - Default to new typhoeus adapter
 
 ## 1.2.1 - 2017-10-30
+
 ### Changed
+
 - Fix issue with Ruby 2.2 compatibility
 
 ## 1.2.0 - 2017-10-30
+
 ### Added
+
 - Add experimental PooledEM adapter (#10)
 
 ### Changed
+
 - Improve marshaling of resources
 
 ## 1.1.0 - 2017-05-12
+
 ### Added
+
 - Add shortcuts for creating fulfilled / rejected promises (#6)
 
 ### Changed
+
 - Return response body if no processor matches (#7)
 
 ## 1.0.0 - 2016-08-22
+
 ### Added
+
 - Experimental cache API doing nothing for now
 
 ### Changed
+
 - Use `~> 1.0` of `concurrent-ruby`
 
 ## 0.5.0 - 2016-04-04
+
 ### Added
+
 - Add `sync` option to typhoeus adapter
 - Add registry for storing entry points
 
 ### Changed
+
 - Make eventmachine based adapter default
 
 ## 0.4.0 - 2016-02-24
+
 ### Added
+
 - Add method to explicit access resource data
 
 ### Changed
+
 - Use typhoeus as default adapter
 - `Restify.new` returns relation now instead of resource
 
 ### Removed
+
 - Drop obligation in favor of simple Concurrent::IVar based promise class.
   Notable changes:
   - Returned object us of type `Restify::Promise` now.

data/lib/restify/adapter/em.rb

@@ -29,7 +29,7 @@ module Restify
         end
 
         # rubocop:disable Style/IdenticalConditionalBranches
-        def call(request, writer, retried = false)
+        def call(request, writer, retried: false)
           if requests.empty?
             requests << [request, writer, retried]
             process_next
@@ -47,10 +47,6 @@ module Restify
           @pipeline
         end
 
-        # rubocop:disable Metrics/AbcSize
-        # rubocop:disable Metrics/CyclomaticComplexity
-        # rubocop:disable Metrics/MethodLength
-        # rubocop:disable Metrics/PerceivedComplexity
         def process_next
           return if requests.empty?
 
@@ -77,7 +73,7 @@ module Restify
               req.last_effective_url,
               req.response_header.status,
               req.response_header,
-              req.response
+              req.response,
             )
 
             if req.response_header['CONNECTION'] == 'close'
@@ -108,7 +104,6 @@ module Restify
             end
           end
         end
-        # rubocop:enable all
       end
 
       def call_native(request, writer)
@@ -128,12 +123,10 @@ module Restify
         return if EventMachine.reactor_running?
 
         Thread.new do
-          begin
-            EventMachine.run {}
-          rescue StandardError => e
-            puts "#{self.class} -> #{e}\n#{e.backtrace.join("\n")}"
-            raise e
-          end
+          EventMachine.run
+        rescue StandardError => e
+          puts "#{self.class} -> #{e}\n#{e.backtrace.join("\n")}"
+          raise e
         end
       end
     end

data/lib/restify/adapter/pooled_em.rb

@@ -63,7 +63,7 @@ module Restify
         #
         def release(conn)
           @available.unshift(conn) if @available.size < @size
-          @used -= 1 if @used > 0
+          @used -= 1 if @used.positive?
 
           logger.debug do
             "[#{conn.uri}] Released to pool (#{@available.size}/#{@used}/#{size})"
@@ -97,7 +97,7 @@ module Restify
         private
 
         def close(conn)
-          @used -= 1 if @used > 0
+          @used -= 1 if @used.positive?
           @host[conn.uri.to_s] -= 1
 
           conn.close
@@ -164,7 +164,7 @@ module Restify
         def new(origin)
           logger.debug do
             "Connect to '#{origin}' " \
-            "(#{@connect_timeout}/#{@inactivity_timeout})..."
+              "(#{@connect_timeout}/#{@inactivity_timeout})..."
           end
 
           @host[origin] += 1
@@ -197,11 +197,10 @@ module Restify
       end
 
       def initialize(**kwargs)
+        super()
         @pool = Pool.new(**kwargs)
       end
 
-      # rubocop:disable Metrics/MethodLength
-      # rubocop:disable Metrics/AbcSize
       # rubocop:disable Metrics/BlockLength
       def call_native(request, writer)
         next_tick do
@@ -212,39 +211,37 @@ module Restify
           end
 
           defer.callback do |conn|
-            begin
-              req = conn.send request.method.downcase,
-                keepalive: true,
-                redirects: 3,
-                path: request.uri.normalized_path,
-                query: request.uri.normalized_query,
-                body: request.body,
-                head: request.headers
-
-              req.callback do
-                writer.fulfill Response.new(
-                  request,
-                  req.last_effective_url,
-                  req.response_header.status,
-                  req.response_header,
-                  req.response
-                )
-
-                if req.response_header['CONNECTION'] == 'close'
-                  @pool.remove(conn)
-                else
-                  @pool << conn
-                end
-              end
-
-              req.errback do
+            req = conn.send request.method.downcase,
+              keepalive: true,
+              redirects: 3,
+              path: request.uri.normalized_path,
+              query: request.uri.normalized_query,
+              body: request.body,
+              head: request.headers
+
+            req.callback do
+              writer.fulfill Response.new(
+                request,
+                req.last_effective_url,
+                req.response_header.status,
+                req.response_header,
+                req.response,
+              )
+
+              if req.response_header['CONNECTION'] == 'close'
                 @pool.remove(conn)
-                writer.reject(req.error)
+              else
+                @pool << conn
               end
-            rescue Exception => e # rubocop:disable Lint/RescueException
+            end
+
+            req.errback do
               @pool.remove(conn)
-              writer.reject(e)
+              writer.reject(req.error)
             end
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            @pool.remove(conn)
+            writer.reject(e)
           end
         end
       end
@@ -261,12 +258,10 @@ module Restify
         return if EventMachine.reactor_running?
 
         Thread.new do
-          begin
-            EventMachine.run {}
-          rescue StandardError => e
-            logger.error(e)
-            raise e
-          end
+          EventMachine.run
+        rescue StandardError => e
+          logger.error(e)
+          raise e
         end
       end
     end

data/lib/restify/adapter/typhoeus.rb

@@ -13,60 +13,52 @@ module Restify
 
       DEFAULT_HEADERS = {
         'Expect' => '',
-        'Transfer-Encoding' => ''
+        'Transfer-Encoding' => '',
       }.freeze
 
       DEFAULT_OPTIONS = {
         followlocation: true,
         tcp_keepalive: true,
         tcp_keepidle: 5,
-        tcp_keepintvl: 5
+        tcp_keepintvl: 5,
       }.freeze
 
       def initialize(sync: false, options: {}, **kwargs)
-        @sync    = sync
         @hydra   = ::Typhoeus::Hydra.new(**kwargs)
         @mutex   = Mutex.new
-        @signal  = ConditionVariable.new
-        @thread  = nil
         @options = DEFAULT_OPTIONS.merge(options)
+        @queue   = Queue.new
+        @sync    = sync
+        @thread  = nil
+
+        super()
       end
 
       def sync?
         @sync
       end
 
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/MethodLength
       def call_native(request, writer)
         req = convert(request, writer)
 
         if sync?
-          req.run
+          @hydra.queue(req)
+          @hydra.run
         else
-          @mutex.synchronize do
-            debug 'request:add',
-              tag: request.object_id,
-              method: request.method.upcase,
-              url: request.uri
-
-            @hydra.queue(req)
-            @hydra.dequeue_many
-
-            thread.run unless thread.status
-          end
+          debug 'request:add',
+            tag: request.object_id,
+            method: request.method.upcase,
+            url: request.uri,
+            timeout: request.timeout
 
-          debug 'request:signal', tag: request.object_id
+          @queue << convert(request, writer)
 
-          @signal.signal
+          thread.run unless thread.status
         end
       end
-      # rubocop:enable all
 
       private
 
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/MethodLength
       def convert(request, writer)
         ::Typhoeus::Request.new(
           request.uri,
@@ -75,25 +67,28 @@ module Restify
           headers: DEFAULT_HEADERS.merge(request.headers),
           body: request.body,
           timeout: request.timeout,
-          connecttimeout: request.timeout
+          connecttimeout: request.timeout,
         ).tap do |req|
           req.on_complete do |response|
             debug 'request:complete',
               tag: request.object_id,
-              status: response.code
+              status: response.code,
+              message: response.return_message,
+              timeout: response.timed_out?
 
-            if response.timed_out?
-              writer.reject Restify::Timeout.new request
-            elsif response.code.zero?
+            if response.timed_out? || response.code.zero?
               writer.reject \
                 Restify::NetworkError.new(request, response.return_message)
             else
               writer.fulfill convert_back(response, request)
             end
+
+            # Add all newly queued requests to active hydra, e.g. requests
+            # queued in a completion callback.
+            dequeue_all
           end
         end
       end
-      # rubocop:enable all
 
       def convert_back(response, request)
         uri     = request.uri
@@ -117,38 +112,49 @@ module Restify
           # Recreate thread if nil or dead
           debug 'hydra:spawn'
 
-          @thread = Thread.new { _loop }
+          @thread = Thread.new do
+            Thread.current.name = 'Restify/Typhoeus Background'
+            run
+          end
         end
 
         @thread
       end
 
-      def _loop
-        Thread.current.name = 'Restify/Typhoeus Background'
-        loop { _run }
-      end
+      def run
+        runs = 0
 
-      def _ongoing?
-        @hydra.queued_requests.any? || @hydra.multi.easy_handles.any?
-      end
+        loop do
+          if @queue.empty? && runs > 100
+            debug 'hydra:gc'
+            GC.start(full_mark: false, immediate_sweep: false)
+            runs = 0
+          end
+
+          debug 'hydra:pop'
 
-      # rubocop:disable Metrics/MethodLength
-      def _run
-        debug 'hydra:run'
-        @hydra.run while _ongoing?
-        debug 'hydra:completed'
+          # Wait for next item and add all available requests to hydra
+          @hydra.queue @queue.pop
+          dequeue_all
 
-        @mutex.synchronize do
-          return if _ongoing?
+          debug 'hydra:run'
+          @hydra.run
+          runs += 1
+          debug 'hydra:completed'
+        rescue StandardError => e
+          logger.error(e)
+        end
+      ensure
+        debug 'hydra:exit'
+      end
 
-          debug 'hydra:pause'
-          @signal.wait(@mutex, 60)
-          debug 'hydra:resumed'
+      def dequeue_all
+        loop do
+          @hydra.queue @queue.pop(true)
+        rescue ThreadError
+          break
         end
-      rescue StandardError => e
-        logger.error(e)
       end
-      # rubocop:enable all
 
       def _log_prefix
         "[#{object_id}/#{Thread.current.object_id}]"

data/lib/restify/context.rb

@@ -47,26 +47,22 @@ module Restify
       processor.new(context, response).resource
     end
 
-    # rubocop:disable Metrics/MethodLength
     def request(method, uri, data: nil, headers: {}, **kwargs)
       request = Request.new(
         headers: default_headers.merge(headers),
         **kwargs,
         method: method,
         uri: join(uri),
-        data: data
+        data: data,
       )
 
       ret = cache.call(request) {|req| adapter.call(req) }
       ret.then do |response|
-        if !response.errored?
-          process response
-        else
-          raise ResponseError.from_code(response)
-        end
+        raise ResponseError.from_code(response) if response.errored?
+
+        process(response)
       end
     end
-    # rubocop:enable all
 
     def encode_with(coder)
       coder.map = marshal_dump
@@ -79,7 +75,7 @@ module Restify
     def marshal_dump
       {
         uri: uri.to_s,
-        headers: default_headers
+        headers: default_headers,
       }
     end
 

data/lib/restify/error.rb

@@ -34,6 +34,8 @@ module Restify
           Gone.new(response)
         when 422
           UnprocessableEntity.new(response)
+        when 429
+          TooManyRequests.new(response)
         when 400...500
           ClientError.new(response)
         when 500
@@ -110,15 +112,37 @@ module Restify
   # This makes it easy to rescue specific expected error types.
 
   class BadRequest < ClientError; end
+
   class Unauthorized < ClientError; end
+
   class NotFound < ClientError; end
+
   class NotAcceptable < ClientError; end
+
   class Gone < ClientError; end
+
   class UnprocessableEntity < ClientError; end
 
+  class TooManyRequests < ClientError
+    def retry_after
+      case response.headers['RETRY_AFTER']
+        when /^\d+$/
+          DateTime.now + Rational(response.headers['RETRY_AFTER'].to_i, 86_400)
+        when String
+          begin
+            DateTime.httpdate response.headers['RETRY_AFTER']
+          rescue ArgumentError
+            nil
+          end
+      end
+    end
+  end
+
   class InternalServerError < ServerError; end
 
   class BadGateway < GatewayError; end
+
   class ServiceUnavailable < GatewayError; end
+
   class GatewayTimeout < GatewayError; end
 end