restify 1.12.0 → 1.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19ee2486506eef195ed64b0a37ff921b97fefa46356e69d2fc3b28ca404ebf98
-  data.tar.gz: 3b81f57ac5b84452dc9e9b6e1bb39705b601138ee5e5f9e2d1bb33d1107e286f
+  metadata.gz: 95d9cf12fd58e2f17bd1d3c046dfda23eaa38ec2fb99fe0c9b7e1034e014cd1f
+  data.tar.gz: b995cf5d935ba1c83b403fedf18d4c35154ea0ad60d4be0437fa98c89281d687
 SHA512:
-  metadata.gz: e2e87e8c7e31b32d7da5f70321887a7ca28831a78e088fe3c934a08dc94481413d4a24c64eb8f765d60c7195a2d9830a11d11df6fa7c9c96ee5033ae26659f80
-  data.tar.gz: 58697191071fa894417566779fcca6f74bb817ae6173bf9d7752b91af0b252d5ae4911fd29507b04fe6adf327e8b73cbf5109509fd9771078936259cb1911e46
+  metadata.gz: 96b5f22cc74ca967876f2ecc86bf30c70fa21ceee6f8726117862ad0e94a6319a9384673cd83a4222bd40b49f4b6eeb48cdec7bd5d1e94ac70fcdab787a8ed5b
+  data.tar.gz: 98199df1ecf1b9467b609ae57f58f2ae459f42f7121c1947763a9d49acb7224d6b8dc50088e74cd7f563719fb81f507d88fca2252a823856e57bbf995db5cbbe

data/CHANGELOG.md

@@ -18,6 +18,42 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Breaks
 
 
+## 1.15.1 - (2021-07-15)
+---
+
+### Fixes
+* Typhoeus internal exception when request timed out
+
+
+## 1.15.0 - (2021-07-09)
+---
+
+### New
+* Improve memory usage when running lots of requests with typhoeus adapter
+* Use hydra for synchronous requests
+* Increased thread stability of typhoeus adapter (new internal queuing mechanism)
+
+### Changes
+* Use Ruby 2.5 as baseline for testing and linting
+* Add Ruby 3.0 to automated testing
+* Changed timing behavior for multiple requests due to new internal queuing mechanism for the typhoeus adapter
+
+
+## 1.14.0 - (2020-12-15)
+---
+
+### New
+* Allow making requests with non-JSON bodies and custom content types (#42)
+
+
+## 1.13.0 - (2020-06-12)
+---
+
+### New
+* typhoeus: Support setting per-request libcurl options on adapter
+* typhoeus: Enable short TCP keepalive probes by default (5s/5s)
+
+
 ## 1.12.0 - (2020-04-01)
 ---
 

data/README.md

@@ -15,20 +15,17 @@ Restify is build upon the following libraries:
 * [addressable](https://github.com/sporkmonger/addressable)
 * [typhoeus](https://github.com/typhoeus/typhoeus)
 
-It has optional HTTP adapters using:
-
-* [em-http-request](https://github.com/igrigorik/em-http-request)
-
 The HTTP adapters are mostly run in a background thread and may not survive mid-application forks.
 
-Included processors can handle:
+Restify includes processors to parse responses and to extract links between resources. The following formats are can be parsed:
 
-* Plain JSON with GitHub-style relations
-* MessagePack with GitHub-style relations *(currently experimental)*
+* JSON
+* MessagePack
 
-(Beside HTTP Link header that's always supported)
+Links are extracted from
 
-Restify requires Ruby 2.0+.
+* HTTP Link header
+* Github-style relations in payloads
 
 ### Planned features
 
@@ -106,9 +103,9 @@ commit = repo.rel(:commits).get.value.first
 And print it:
 
 ```ruby
-puts "Last commit: #{commit[:sha]}"
-puts "By #{commit[:commit][:author][:name]} <#{commit[:commit][:author][:email]}>"
-puts "#{commit[:commit][:message]}"
+puts "Last commit: #{commit['sha']}"
+puts "By #{commit['commit']['author']['name']} <#{commit['commit']['author']['email']}>"
+puts "#{commit['commit']['message']}"
 ```
 
 See commented example in main spec [`spec/restify_spec.rb`](https://github.com/jgraichen/restify/blob/master/spec/restify_spec.rb#L100) or in the `examples` directory.

data/lib/restify/adapter/em.rb

@@ -63,8 +63,8 @@ module Restify
               query: request.uri.normalized_query,
               body: request.body,
               head: request.headers
-          rescue Exception => err # rubocop:disable RescueException
-            writer.reject err
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            writer.reject e
             requests.shift unless pipeline?
             return
           end
@@ -128,12 +128,10 @@ module Restify
         return if EventMachine.reactor_running?
 
         Thread.new do
-          begin
-            EventMachine.run {}
-          rescue StandardError => e
-            puts "#{self.class} -> #{e}\n#{e.backtrace.join("\n")}"
-            raise e
-          end
+          EventMachine.run {}
+        rescue StandardError => e
+          puts "#{self.class} -> #{e}\n#{e.backtrace.join("\n")}"
+          raise e
         end
       end
     end

data/lib/restify/adapter/pooled_em.rb

@@ -63,7 +63,7 @@ module Restify
         #
         def release(conn)
           @available.unshift(conn) if @available.size < @size
-          @used -= 1 if @used > 0
+          @used -= 1 if @used.positive?
 
           logger.debug do
             "[#{conn.uri}] Released to pool (#{@available.size}/#{@used}/#{size})"
@@ -97,7 +97,7 @@ module Restify
         private
 
         def close(conn)
-          @used -= 1 if @used > 0
+          @used -= 1 if @used.positive?
           @host[conn.uri.to_s] -= 1
 
           conn.close
@@ -200,9 +200,9 @@ module Restify
         @pool = Pool.new(**kwargs)
       end
 
-      # rubocop:disable MethodLength
-      # rubocop:disable AbcSize
-      # rubocop:disable BlockLength
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/BlockLength
       def call_native(request, writer)
         next_tick do
           defer = @pool.get(request)
@@ -212,39 +212,37 @@ module Restify
           end
 
           defer.callback do |conn|
-            begin
-              req = conn.send request.method.downcase,
-                keepalive: true,
-                redirects: 3,
-                path: request.uri.normalized_path,
-                query: request.uri.normalized_query,
-                body: request.body,
-                head: request.headers
-
-              req.callback do
-                writer.fulfill Response.new(
-                  request,
-                  req.last_effective_url,
-                  req.response_header.status,
-                  req.response_header,
-                  req.response
-                )
-
-                if req.response_header['CONNECTION'] == 'close'
-                  @pool.remove(conn)
-                else
-                  @pool << conn
-                end
-              end
-
-              req.errback do
+            req = conn.send request.method.downcase,
+              keepalive: true,
+              redirects: 3,
+              path: request.uri.normalized_path,
+              query: request.uri.normalized_query,
+              body: request.body,
+              head: request.headers
+
+            req.callback do
+              writer.fulfill Response.new(
+                request,
+                req.last_effective_url,
+                req.response_header.status,
+                req.response_header,
+                req.response
+              )
+
+              if req.response_header['CONNECTION'] == 'close'
                 @pool.remove(conn)
-                writer.reject(req.error)
+              else
+                @pool << conn
               end
-            rescue Exception => ex # rubocop:disable RescueException
+            end
+
+            req.errback do
               @pool.remove(conn)
-              writer.reject(ex)
+              writer.reject(req.error)
             end
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            @pool.remove(conn)
+            writer.reject(e)
           end
         end
       end
@@ -261,12 +259,10 @@ module Restify
         return if EventMachine.reactor_running?
 
         Thread.new do
-          begin
-            EventMachine.run {}
-          rescue StandardError => e
-            logger.error(e)
-            raise e
-          end
+          EventMachine.run {}
+        rescue StandardError => e
+          logger.error(e)
+          raise e
         end
       end
     end

data/lib/restify/adapter/typhoeus.rb

@@ -16,76 +16,85 @@ module Restify
         'Transfer-Encoding' => ''
       }.freeze
 
-      def initialize(sync: false, **options)
-        @sync   = sync
-        @hydra  = ::Typhoeus::Hydra.new(**options)
-        @mutex  = Mutex.new
-        @signal = ConditionVariable.new
-        @thread = nil
+      DEFAULT_OPTIONS = {
+        followlocation: true,
+        tcp_keepalive: true,
+        tcp_keepidle: 5,
+        tcp_keepintvl: 5
+      }.freeze
+
+      def initialize(sync: false, options: {}, **kwargs)
+        @hydra   = ::Typhoeus::Hydra.new(**kwargs)
+        @mutex   = Mutex.new
+        @options = DEFAULT_OPTIONS.merge(options)
+        @queue   = Queue.new
+        @sync    = sync
+        @thread  = nil
+
+        super()
       end
 
       def sync?
         @sync
       end
 
-      # rubocop:disable AbcSize
-      # rubocop:disable MethodLength
+      # rubocop:disable Metrics/MethodLength
       def call_native(request, writer)
         req = convert(request, writer)
 
         if sync?
-          req.run
+          @hydra.queue(req)
+          @hydra.run
         else
-          @mutex.synchronize do
-            debug 'request:add',
-              tag: request.object_id,
-              method: request.method.upcase,
-              url: request.uri
+          debug 'request:add',
+            tag: request.object_id,
+            method: request.method.upcase,
+            url: request.uri,
+            timeout: request.timeout
 
-            @hydra.queue(req)
-            @hydra.dequeue_many
+          @queue << convert(request, writer)
 
-            thread.run unless thread.status
-          end
-
-          debug 'request:signal', tag: request.object_id
-
-          @signal.signal
+          thread.run unless thread.status
         end
       end
-      # rubocop:enable all
+      # rubocop:enable Metrics/MethodLength
 
       private
 
-      # rubocop:disable AbcSize
-      # rubocop:disable MethodLength
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
       def convert(request, writer)
         ::Typhoeus::Request.new(
           request.uri,
+          **@options,
           method: request.method,
           headers: DEFAULT_HEADERS.merge(request.headers),
           body: request.body,
-          followlocation: true,
           timeout: request.timeout,
           connecttimeout: request.timeout
         ).tap do |req|
           req.on_complete do |response|
             debug 'request:complete',
               tag: request.object_id,
-              status: response.code
+              status: response.code,
+              message: response.return_message,
+              timeout: response.timed_out?
 
-            if response.timed_out?
-              writer.reject Restify::Timeout.new request
-            elsif response.code.zero?
+            if response.timed_out? || response.code.zero?
               writer.reject \
                 Restify::NetworkError.new(request, response.return_message)
             else
               writer.fulfill convert_back(response, request)
             end
+
+            # Add all newly queued requests to active hydra, e.g. requests
+            # queued in a completion callback.
+            dequeue_all
           end
         end
       end
-      # rubocop:enable all
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
 
       def convert_back(response, request)
         uri     = request.uri
@@ -109,38 +118,51 @@ module Restify
           # Recreate thread if nil or dead
           debug 'hydra:spawn'
 
-          @thread = Thread.new { _loop }
+          @thread = Thread.new do
+            Thread.current.name = 'Restify/Typhoeus Background'
+            run
+          end
         end
 
         @thread
       end
 
-      def _loop
-        Thread.current.name = 'Restify/Typhoeus Background'
-        loop { _run }
-      end
+      # rubocop:disable Metrics/MethodLength
+      def run
+        runs = 0
 
-      def _ongoing?
-        @hydra.queued_requests.any? || @hydra.multi.easy_handles.any?
-      end
+        loop do
+          if @queue.empty? && runs > 100
+            debug 'hydra:gc'
+            GC.start(full_mark: false, immediate_sweep: false)
+            runs = 0
+          end
 
-      # rubocop:disable MethodLength
-      def _run
-        debug 'hydra:run'
-        @hydra.run while _ongoing?
-        debug 'hydra:completed'
+          debug 'hydra:pop'
 
-        @mutex.synchronize do
-          return if _ongoing?
+          # Wait for next item and add all available requests to hydra
+          @hydra.queue @queue.pop
+          dequeue_all
+
+          debug 'hydra:run'
+          @hydra.run
+          runs += 1
+          debug 'hydra:completed'
+        rescue StandardError => e
+          logger.error(e)
+        end
+      ensure
+        debug 'hydra:exit'
+      end
+      # rubocop:enable Metrics/MethodLength
 
-          debug 'hydra:pause'
-          @signal.wait(@mutex, 60)
-          debug 'hydra:resumed'
+      def dequeue_all
+        loop do
+          @hydra.queue @queue.pop(true)
+        rescue ThreadError
+          break
         end
-      rescue StandardError => e
-        logger.error(e)
       end
-      # rubocop:enable all
 
       def _log_prefix
         "[#{object_id}/#{Thread.current.object_id}]"

data/lib/restify/context.rb

@@ -36,7 +36,7 @@ module Restify
 
     def inherit(uri, **kwargs)
       uri ||= self.uri
-      Context.new uri, kwargs.merge(options)
+      Context.new(uri, **kwargs, **options)
     end
 
     def process(response)
@@ -59,10 +59,10 @@ module Restify
 
       ret = cache.call(request) {|req| adapter.call(req) }
       ret.then do |response|
-        if !response.errored?
-          process response
-        else
+        if response.errored?
           raise ResponseError.from_code(response)
+        else
+          process response
         end
       end
     end