restify 1.10.0 → 1.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f44b84123c73ffd08cc8668448b03470c06acaf0f693e844fface91ff44179b5
-  data.tar.gz: c194b2db48a18a7f234bba20ff4161f98277b1146e6b7bf2572c877c285c3794
+  metadata.gz: 241fd43c8e40d2a0c2c839443df03106edf3c8e491cac9338277ff4327a3927f
+  data.tar.gz: d8eea64a1de42ee6eb4666feb510669b69482f24c72a0f5c883f3482143751ca
 SHA512:
-  metadata.gz: 89de3a2282568b6cd4b58afdf04e2271abf283721a25535e5420a4b1e52e4ed6b3edf894180d6fd15ee010c7692f99942c047fe7baef81d5bd67ff13b756e189
-  data.tar.gz: e6b18bfd819aec933acdb9fbec820ea7e742918eebc795c77f5f7ba0b927ccae85b876b367df8325d90e2899aad77bcf30f691781419669fd7406a156e197f86
+  metadata.gz: 603395d24a3ecad6d35b217ef2d154cd0982f9cd9181769b2a5aa949e134f0f9af1a15cc731f54a2d700e3bd1162b667240420e5669af20b48c66a1ba84fc2c5
+  data.tar.gz: 89e64dcd00dde5a7711922619380d2003db5f8c42a7cbc473b19678b6ed251134ffe4b4c94b86afaa7efa7dea030c6d43876e57c00c85ea76523f9f7029f6804

data/CHANGELOG.md

@@ -1,91 +1,173 @@
 # Changelog
+All notable changes to this project will be documented in this file.
 
-## 1.10.0
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-* Raise more specific error on a few status codes (#17)
-* Complete promises with an empty list (but a list) of dependencies (#18)
 
-## 1.9.0
 
-* Do not raise error on 3XX responses but return responses
+## Unreleased
+---
 
-## 1.8.0
+### New
 
-* Add HEAD request method (#16)
+### Changes
 
-## 1.7.0
+### Fixes
 
-* Introduce promise dependency timeouts (#15)
+### Breaks
 
-## 1.6.0
 
-* Specify headers on restify clients and individual requests (#14)
+## 1.15.0 - (2021-07-09)
+---
 
-## 1.5.0
+### New
+* Improve memory usage when running lots of requests with typhoeus adapter
+* Use hydra for synchronous requests
+* Increased thread stability of typhoeus adapter (new internal queuing mechanism)
 
-* Tune typhoeus adapter to be more race-condition resilent
-* Add MessagePack processor enabled by default
+### Changes
+* Use Ruby 2.5 as baseline for testing and linting
+* Add Ruby 3.0 to automated testing
+* Changed timing behavior for multiple requests due to new internal queuing mechanism for the typhoeus adapter
 
-## 1.4.4
 
-* Fix race condition in typhoeus adapter
-* Add `#request` to `NetworkError` to ease debugging
+## 1.14.0 - (2020-12-15)
 
-## 1.4.3
+### New
+* Allow making requests with non-JSON bodies and custom content types (#42)
 
-* Add advanced logging capabilities using logging gem
-* Improve compatibility with webmocks returning `nil` as headers
 
-## 1.4.1
+## 1.13.0 - (2020-06-12)
+---
 
-* Fix possible deadlock issues
+### New
+* typhoeus: Support setting per-request libcurl options on adapter
+* typhoeus: Enable short TCP keepalive probes by default (5s/5s)
 
-## 1.4.0
 
-* Fix possible concurrency issue with typhoeus adapter
-* Add timeout option to requests (only supported by typhoeus adapter)
+## 1.12.0 - (2020-04-01)
+---
 
-## 1.3.1
+### Added
+* Explicit exception class for HTTP status code 410 (Gone)
 
-* Improve typhoeus adapters initial request queuing
-* Disable default pipelining
+### Changed
 
-## 1.3.0
+### Fixed
+* `GatewayError` exception classes introduced in v1.11.0 now properly inherit from `ServerError` (#30)
 
-* Improve typhoeus adapter to better utilize concurrency
-* Default to new typhoeus adapter
 
-## 1.2.1
+## 1.11.0 - (2019-07-11)
+### Added
+* Explicit exception classes for HTTP status codes 500, 502, 503, 504
 
-* Fix issue with Ruby 2.2 compatibility
+## 1.10.0 - 2018-12-11
+### Changed
+- Raise more specific error on a few status codes (#17)
+- Complete promises with an empty list (but a list) of dependencies (#18)
 
-## 1.2.0
+## 1.9.0 - 2018-11-13
+### Changed
+- Do not raise error on 3XX responses but return responses
 
-* Add experimental PooledEM adapter (#10)
-* Improve marshalling of resources
+## 1.8.0 - 2018-08-22
+### Added
+- Add HEAD request method (#16)
 
-## 1.1.0
+## 1.7.0 - 2018-08-15
+### Added
+- Introduce promise dependency timeouts (#15)
 
-* Return response body if no processor matches (#7)
-* Add shortcuts for creating fulfilled / rejected promises (#6)
+## 1.6.0 - 2018-08-09
+### Changed
+- Specify headers on restify clients and individual requests (#14)
 
-## 1.0.0
+## 1.5.0 - 2018-07-31
+### Added
+- Add MessagePack processor enabled by default
 
-* Experimental cache API doing nothing for now
-* Use `~> 1.0` of `concurrent-ruby`
+### Changed
+- Tune typhoeus adapter to be more race-condition resilent
 
-## 0.5.0
+## 1.4.4 - 2018-07-13
+### Added
+- Add `#request` to `NetworkError` to ease debugging
 
-* Add `sync` option to typhoeus adapter
-* Add registry for storing entry points
-* Make eventmachine based adapter default
+### Changed
+- Fix race condition in typhoeus adapter
 
-## 0.4.0
+## 1.4.3 - 2017-11-15
+### Added
+- Add advanced logging capabilities using logging gem
 
-* Add method to explicit access resource data
-* Drop obligation in favor of simple Concurrent::IVar based promise class.
+### Changed
+- Improve compatibility with webmocks returning `nil` as headers
+
+## 1.4.1 - 2017-11-15
+### Changed
+- Fix possible deadlock issues
+
+## 1.4.0 - 2017-11-10
+### Added
+- Add timeout option to requests (only supported by typhoeus adapter)
+
+### Changed
+- Fix possible concurrency issue with typhoeus adapter
+
+## 1.3.1 - 2017-11-10
+### Changed
+- Improve typhoeus adapters initial request queuing
+- Disable default pipelining
+
+## 1.3.0 - 2017-11-08
+### Changed
+- Improve typhoeus adapter to better utilize concurrency
+- Default to new typhoeus adapter
+
+## 1.2.1 - 2017-10-30
+### Changed
+- Fix issue with Ruby 2.2 compatibility
+
+## 1.2.0 - 2017-10-30
+### Added
+- Add experimental PooledEM adapter (#10)
+
+### Changed
+- Improve marshaling of resources
+
+## 1.1.0 - 2017-05-12
+### Added
+- Add shortcuts for creating fulfilled / rejected promises (#6)
+
+### Changed
+- Return response body if no processor matches (#7)
+
+## 1.0.0 - 2016-08-22
+### Added
+- Experimental cache API doing nothing for now
+
+### Changed
+- Use `~> 1.0` of `concurrent-ruby`
+
+## 0.5.0 - 2016-04-04
+### Added
+- Add `sync` option to typhoeus adapter
+- Add registry for storing entry points
+
+### Changed
+- Make eventmachine based adapter default
+
+## 0.4.0 - 2016-02-24
+### Added
+- Add method to explicit access resource data
+
+### Changed
+- Use typhoeus as default adapter
+- `Restify.new` returns relation now instead of resource
+
+### Removed
+- Drop obligation in favor of simple Concurrent::IVar based promise class.
   Notable changes:
-    - Returned object us of type `Restify::Promise` now.
-    - `value` will not raise exception but return `nil` in case of failure. Use `value!` for old behavior.
-* Use typhoeus as default adapter
-* `Restify.new` returns relation now instead of resource
+  - Returned object us of type `Restify::Promise` now.
+  - `value` will not raise exception but return `nil` in case of failure. Use `value!` for old behavior.

data/README.md

@@ -1,6 +1,7 @@
 # Restify
 
 [![Build Status](https://travis-ci.org/jgraichen/restify.svg?branch=master)](https://travis-ci.org/jgraichen/restify)
+[![Code Quality](https://codebeat.co/badges/18ffe6b7-8239-493a-b5b6-be329b9f275d)](https://codebeat.co/projects/github-com-jgraichen-restify-master)
 
 Restify is an hypermedia REST client that does parallel, concurrent and keep-alive requests by default.
 
@@ -14,20 +15,17 @@ Restify is build upon the following libraries:
 * [addressable](https://github.com/sporkmonger/addressable)
 * [typhoeus](https://github.com/typhoeus/typhoeus)
 
-It has optional HTTP adapters using:
-
-* [em-http-request](https://github.com/igrigorik/em-http-request)
-
 The HTTP adapters are mostly run in a background thread and may not survive mid-application forks.
 
-Included processors can handle:
+Restify includes processors to parse responses and to extract links between resources. The following formats are can be parsed:
 
-* Plain JSON with GitHub-style relations
-* MessagePack with GitHub-style relations *(currently experimental)*
+* JSON
+* MessagePack
 
-(Beside HTTP Link header that's always supported)
+Links are extracted from
 
-Restify requires Ruby 2.0+.
+* HTTP Link header
+* Github-style relations in payloads
 
 ### Planned features
 
@@ -105,9 +103,9 @@ commit = repo.rel(:commits).get.value.first
 And print it:
 
 ```ruby
-puts "Last commit: #{commit[:sha]}"
-puts "By #{commit[:commit][:author][:name]} <#{commit[:commit][:author][:email]}>"
-puts "#{commit[:commit][:message]}"
+puts "Last commit: #{commit['sha']}"
+puts "By #{commit['commit']['author']['name']} <#{commit['commit']['author']['email']}>"
+puts "#{commit['commit']['message']}"
 ```
 
 See commented example in main spec [`spec/restify_spec.rb`](https://github.com/jgraichen/restify/blob/master/spec/restify_spec.rb#L100) or in the `examples` directory.

data/lib/restify/adapter/em.rb

@@ -63,8 +63,8 @@ module Restify
               query: request.uri.normalized_query,
               body: request.body,
               head: request.headers
-          rescue Exception => err # rubocop:disable RescueException
-            writer.reject err
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            writer.reject e
             requests.shift unless pipeline?
             return
           end
@@ -128,12 +128,10 @@ module Restify
         return if EventMachine.reactor_running?
 
         Thread.new do
-          begin
-            EventMachine.run {}
-          rescue StandardError => e
-            puts "#{self.class} -> #{e}\n#{e.backtrace.join("\n")}"
-            raise e
-          end
+          EventMachine.run {}
+        rescue StandardError => e
+          puts "#{self.class} -> #{e}\n#{e.backtrace.join("\n")}"
+          raise e
         end
       end
     end

data/lib/restify/adapter/pooled_em.rb

@@ -63,7 +63,7 @@ module Restify
         #
         def release(conn)
           @available.unshift(conn) if @available.size < @size
-          @used -= 1 if @used > 0
+          @used -= 1 if @used.positive?
 
           logger.debug do
             "[#{conn.uri}] Released to pool (#{@available.size}/#{@used}/#{size})"
@@ -97,7 +97,7 @@ module Restify
         private
 
         def close(conn)
-          @used -= 1 if @used > 0
+          @used -= 1 if @used.positive?
           @host[conn.uri.to_s] -= 1
 
           conn.close
@@ -200,9 +200,9 @@ module Restify
         @pool = Pool.new(**kwargs)
       end
 
-      # rubocop:disable MethodLength
-      # rubocop:disable AbcSize
-      # rubocop:disable BlockLength
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/BlockLength
       def call_native(request, writer)
         next_tick do
           defer = @pool.get(request)
@@ -212,39 +212,37 @@ module Restify
           end
 
           defer.callback do |conn|
-            begin
-              req = conn.send request.method.downcase,
-                keepalive: true,
-                redirects: 3,
-                path: request.uri.normalized_path,
-                query: request.uri.normalized_query,
-                body: request.body,
-                head: request.headers
-
-              req.callback do
-                writer.fulfill Response.new(
-                  request,
-                  req.last_effective_url,
-                  req.response_header.status,
-                  req.response_header,
-                  req.response
-                )
-
-                if req.response_header['CONNECTION'] == 'close'
-                  @pool.remove(conn)
-                else
-                  @pool << conn
-                end
-              end
-
-              req.errback do
+            req = conn.send request.method.downcase,
+              keepalive: true,
+              redirects: 3,
+              path: request.uri.normalized_path,
+              query: request.uri.normalized_query,
+              body: request.body,
+              head: request.headers
+
+            req.callback do
+              writer.fulfill Response.new(
+                request,
+                req.last_effective_url,
+                req.response_header.status,
+                req.response_header,
+                req.response
+              )
+
+              if req.response_header['CONNECTION'] == 'close'
                 @pool.remove(conn)
-                writer.reject(req.error)
+              else
+                @pool << conn
               end
-            rescue Exception => ex # rubocop:disable RescueException
+            end
+
+            req.errback do
               @pool.remove(conn)
-              writer.reject(ex)
+              writer.reject(req.error)
             end
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            @pool.remove(conn)
+            writer.reject(e)
           end
         end
       end
@@ -261,12 +259,10 @@ module Restify
         return if EventMachine.reactor_running?
 
         Thread.new do
-          begin
-            EventMachine.run {}
-          rescue StandardError => e
-            logger.error(e)
-            raise e
-          end
+          EventMachine.run {}
+        rescue StandardError => e
+          logger.error(e)
+          raise e
         end
       end
     end

data/lib/restify/adapter/typhoeus.rb

@@ -16,55 +16,59 @@ module Restify
         'Transfer-Encoding' => ''
       }.freeze
 
-      def initialize(sync: false, **options)
-        @sync   = sync
-        @hydra  = ::Typhoeus::Hydra.new(**options)
-        @mutex  = Mutex.new
-        @signal = ConditionVariable.new
+      DEFAULT_OPTIONS = {
+        followlocation: true,
+        tcp_keepalive: true,
+        tcp_keepidle: 5,
+        tcp_keepintvl: 5
+      }.freeze
+
+      def initialize(sync: false, options: {}, **kwargs)
+        @hydra   = ::Typhoeus::Hydra.new(**kwargs)
+        @mutex   = Mutex.new
+        @options = DEFAULT_OPTIONS.merge(options)
+        @queue   = Queue.new
+        @sync    = sync
+        @thread  = nil
+
+        super()
       end
 
       def sync?
         @sync
       end
 
-      # rubocop:disable AbcSize
-      # rubocop:disable MethodLength
+      # rubocop:disable Metrics/MethodLength
       def call_native(request, writer)
         req = convert(request, writer)
 
         if sync?
-          req.run
+          @hydra.queue(req)
+          @hydra.run
         else
-          @mutex.synchronize do
-            debug 'request:add',
-              tag: request.object_id,
-              method: request.method.upcase,
-              url: request.uri
+          debug 'request:add',
+            tag: request.object_id,
+            method: request.method.upcase,
+            url: request.uri
 
-            @hydra.queue(req)
-            @hydra.dequeue_many
+          @queue << convert(request, writer)
 
-            thread.run unless thread.status
-          end
-
-          debug 'request:signal', tag: request.object_id
-
-          @signal.signal
+          thread.run unless thread.status
         end
       end
-      # rubocop:enable all
+      # rubocop:enable Metrics/MethodLength
 
       private
 
-      # rubocop:disable AbcSize
-      # rubocop:disable MethodLength
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
       def convert(request, writer)
         ::Typhoeus::Request.new(
           request.uri,
+          **@options,
           method: request.method,
           headers: DEFAULT_HEADERS.merge(request.headers),
           body: request.body,
-          followlocation: true,
           timeout: request.timeout,
           connecttimeout: request.timeout
         ).tap do |req|
@@ -81,10 +85,15 @@ module Restify
             else
               writer.fulfill convert_back(response, request)
             end
+
+            # Add all newly queued requests to active hydra, e.g. requests
+            # queued in a completion callback.
+            dequeue_all
           end
         end
       end
-      # rubocop:enable all
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
 
       def convert_back(response, request)
         uri     = request.uri
@@ -108,38 +117,51 @@ module Restify
           # Recreate thread if nil or dead
           debug 'hydra:spawn'
 
-          @thread = Thread.new { _loop }
+          @thread = Thread.new do
+            Thread.current.name = 'Restify/Typhoeus Background'
+            run
+          end
         end
 
         @thread
       end
 
-      def _loop
-        Thread.current.name = 'Restify/Typhoeus Background'
-        loop { _run }
-      end
+      # rubocop:disable Metrics/MethodLength
+      def run
+        runs = 0
 
-      def _ongoing?
-        @hydra.queued_requests.any? || @hydra.multi.easy_handles.any?
-      end
+        loop do
+          if @queue.empty? && runs > 100
+            debug 'hydra:gc'
+            GC.start(full_mark: false, immediate_sweep: false)
+            runs = 0
+          end
 
-      # rubocop:disable MethodLength
-      def _run
-        debug 'hydra:run'
-        @hydra.run while _ongoing?
-        debug 'hydra:completed'
+          debug 'hydra:pop'
 
-        @mutex.synchronize do
-          return if _ongoing?
+          # Wait for next item and add all available requests to hydra
+          @hydra.queue @queue.pop
+          dequeue_all
+
+          debug 'hydra:run'
+          @hydra.run
+          runs += 1
+          debug 'hydra:completed'
+        rescue StandardError => e
+          logger.error(e)
+        end
+      ensure
+        debug 'hydra:exit'
+      end
+      # rubocop:enable Metrics/MethodLength
 
-          debug 'hydra:pause'
-          @signal.wait(@mutex, 60)
-          debug 'hydra:resumed'
+      def dequeue_all
+        loop do
+          @hydra.queue @queue.pop(true)
+        rescue ThreadError
+          break
         end
-      rescue StandardError => e
-        logger.error(e)
       end
-      # rubocop:enable all
 
       def _log_prefix
         "[#{object_id}/#{Thread.current.object_id}]"