restify 2.1.0 → 3.0.0.rc2

data/lib/restify/response.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'rack/media_type'
 require 'rack/utils'
 require 'json'
 
@@ -30,6 +31,10 @@ module Restify
 
     # Response body as string.
     #
+    # The body is encoded according to the charset from the response's
+    # `Content-Type` header. Without a charset, or with one unknown to
+    # Ruby, the body is returned as binary.
+    #
     # @return [String] Response body.
     #
     attr_reader :body
@@ -81,7 +86,7 @@ module Restify
       @code    = code
       @status  = STATUS_CODE_TO_SYMBOL[code]
       @headers = convert_headers(headers)
-      @body    = body
+      @body    = encode_body(body)
       @message = Rack::Utils::HTTP_STATUS_CODES[code]
     end
 
@@ -112,6 +117,24 @@ module Restify
       headers['CONTENT_TYPE']
     end
 
+    # Return the encoding from the response's content type, if any.
+    #
+    # @return [Encoding, nil] Encoding, or nil if the content type
+    #   carries no charset or an encoding unknown to Ruby.
+    #
+    def charset
+      return @charset if defined?(@charset)
+
+      charset = Rack::MediaType.params(content_type.to_s)['charset']
+
+      @charset = begin
+        Encoding.find(charset) if charset
+      rescue ArgumentError
+        # Unknown: keep the body as binary rather than guessing.
+        nil
+      end
+    end
+
     # Check if response is successful e.g. the status code
     # is on of 2XX.
     #
@@ -130,13 +153,20 @@ module Restify
       (400...600).cover? code
     end
 
+    # Decoded body for error messages, or nil if the body is empty or
+    # could not be parsed.
+    #
     # @api private
     def decoded_body
-      @decoded_body ||= begin
+      return @decoded_body if defined?(@decoded_body)
+
+      @decoded_body = begin
         case content_type
           when %r{\Aapplication/json($|;)}
-            ::JSON.parse(body)
+            ::JSON.parse(body) unless body.nil? || body.empty?
         end
+      rescue ::JSON::ParserError
+        nil
       end
     end
 
@@ -147,6 +177,12 @@ module Restify
 
     private
 
+    def encode_body(body)
+      return body unless body && charset
+
+      (+body).force_encoding(charset)
+    end
+
     def convert_headers(headers)
       headers.each.to_h do |pair|
         [pair[0].upcase, pair[1]]

data/lib/restify/timeout.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'hitimes'
+require 'timeout'
 
 module Restify
   class Timeout
@@ -14,9 +14,7 @@ module Restify
     def initialize(timeout, target = nil)
       @timeout = parse_timeout(timeout)
       @target = target
-
-      @interval = ::Hitimes::Interval.new
-      @interval.start
+      @deadline = now + @timeout
     end
 
     def wait_on!(ivar)
@@ -31,6 +29,10 @@ module Restify
       raise self if wait_interval <= 0
     end
 
+    def remaining
+      wait_interval
+    end
+
     def exception
       Error.new(@target)
     end
@@ -38,7 +40,11 @@ module Restify
     private
 
     def wait_interval
-      @timeout - @interval.to_f
+      @deadline - now
+    end
+
+    def now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
 
     def parse_timeout(value)

data/lib/restify/version.rb

@@ -2,10 +2,10 @@
 
 module Restify
   module VERSION
-    MAJOR = 2
-    MINOR = 1
+    MAJOR = 3
+    MINOR = 0
     PATCH = 0
-    STAGE = nil
+    STAGE = :rc2
     STRING = [MAJOR, MINOR, PATCH, STAGE].compact.join('.').freeze
 
     def self.to_s

data/lib/restify/webmock.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'restify'
+require 'webmock'
+
+module WebMock
+  module HttpLibAdapters
+    #
+    # WebMock support for Restify (independent of the adapter)
+    #
+    # Requests are intercepted before they are handed over to an
+    # adapter. Stubbed responses are returned as they are, including
+    # redirects, like WebMock does for other libcurl-based libraries.
+    # Requests not stubbed are passed on to the adapter, if allowed.
+    #
+    # Must be loaded before WebMock is enabled, e.g. in
+    # `spec_helper.rb`:
+    #
+    #     require 'webmock/rspec'
+    #     require 'restify/webmock'
+    #
+    class RestifyAdapter < HttpLibAdapter
+      adapter_for :restify
+
+      class << self
+        def enable!
+          @enabled = true
+        end
+
+        def disable!
+          @enabled = false
+        end
+
+        def enabled?
+          @enabled == true
+        end
+
+        # Respond to a request with a stubbed response, or pass it on to
+        # the given block if allowed.
+        #
+        # @api private
+        def call(request, &)
+          signature = signature(request)
+          RequestRegistry.instance.requested_signatures.put(signature)
+
+          stub = StubRegistry.instance.response_for_request(signature)
+          return passthrough(signature, &) unless stub
+
+          CallbackRegistry.invoke_callbacks({lib: :restify}, signature, stub)
+
+          Restify::Promise.fulfilled(respond(request, stub))
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          # e.g. `NetConnectNotAllowedError`
+          Restify::Promise.rejected(e)
+        end
+
+        private
+
+        def signature(request)
+          RequestSignature.new(
+            request.method.to_sym,
+            request.uri.to_s,
+            body: request.body,
+            headers: request.headers,
+          )
+        end
+
+        def passthrough(signature)
+          unless ::WebMock.net_connect_allowed?(signature.uri)
+            raise NetConnectNotAllowedError.new(signature)
+          end
+
+          yield.tap do |promise|
+            # Observers run when the request is complete, even when
+            # nobody waits on the promise.
+            promise.add_observer do |_, response, _|
+              next unless response
+
+              CallbackRegistry.invoke_callbacks(
+                {lib: :restify, real_request: true},
+                signature,
+                webmock_response(response),
+              )
+            end
+          end
+        end
+
+        def respond(request, stub)
+          raise Restify::NetworkError.new(request, 'Timeout was reached') if stub.should_timeout
+
+          stub.raise_error_if_any
+
+          headers = (stub.headers || {}).to_h do |name, value|
+            [name.upcase.tr('-', '_'), value]
+          end
+
+          Restify::Response.new(request, request.uri, stub.status[0], headers, stub.body)
+        end
+
+        def webmock_response(response)
+          Response.new.tap do |webmock|
+            webmock.status = [response.code, response.message.to_s]
+            webmock.headers = response.headers.to_h do |name, value|
+              [name.split('_').map(&:capitalize).join('-'), value]
+            end
+            webmock.body = response.body.to_s
+          end
+        end
+      end
+
+      # Intercept requests after `Adapter::Telemetry` but before
+      # `Adapter::Base#call` so that telemetry is included.
+      #
+      # @api private
+      module Intercept
+        def call(request)
+          return super unless RestifyAdapter.enabled?
+
+          RestifyAdapter.call(request) { super }
+        end
+      end
+
+      ::Restify::Adapter::Telemetry.include(Intercept)
+    end
+  end
+end

data/lib/restify.rb

@@ -28,7 +28,11 @@ module Restify
 
   module Adapter
     require 'restify/adapter/base'
-    require 'restify/adapter/typhoeus'
+    require 'restify/adapter/loop_lock'
+    require 'restify/adapter/ethon'
+
+    # Optional, requires the `typhoeus` gem.
+    autoload :Typhoeus, 'restify/adapter/typhoeus'
   end
 
   module Processors

data/spec/restify/adapter/ethon_spec.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Restify::Adapter::Ethon do
+  describe 'completed transfers' do
+    subject(:value) do
+      Restify::Promise.create do |writer|
+        described_class.new.send(
+          :complete,
+          easy,
+          request,
+          writer,
+        )
+      end.value!
+    end
+
+    let(:request) { Restify::Request.new(uri: 'http://example.org/base') }
+    let(:return_code) { :ok }
+    let(:response_code) { 200 }
+    let(:effective_url) { 'http://example.org/base' }
+
+    # A transfer is handed back from libcurl with its return code and,
+    # for HTTP, the status code of the response.
+    def easy
+      instance_double(
+        described_class::Easy,
+        return_code:,
+        response_code:,
+        effective_url:,
+        response_headers: "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n",
+        response_body: '{}',
+      )
+    end
+
+    it 'fulfills the promise with the response' do
+      expect(value.code).to eq 200
+      expect(value.uri.to_s).to eq 'http://example.org/base'
+    end
+
+    context 'when redirected' do
+      let(:effective_url) { 'http://example.org/other/base' }
+
+      it 'uses the effective URL as the response URI' do
+        expect(value.uri.to_s).to eq 'http://example.org/other/base'
+      end
+    end
+
+    context 'when the transfer failed' do
+      let(:return_code) { :couldnt_connect }
+      let(:response_code) { 0 }
+
+      it 'rejects the promise' do
+        expect { value }.to raise_error Restify::NetworkError, /connect/i
+      end
+    end
+
+    # Non-HTTP protocols are refused before a transfer is started, but
+    # libcurl still reports success with a zero status code whenever no
+    # HTTP status was received, and no status code at all when it cannot
+    # be read back. Neither has a response to process.
+    context 'without an HTTP status' do
+      let(:response_code) { 0 }
+
+      it 'rejects the promise' do
+        expect { value }.to raise_error Restify::NetworkError, /without HTTP status/
+      end
+    end
+
+    context 'with an unavailable HTTP status' do
+      let(:response_code) { nil }
+
+      it 'rejects the promise' do
+        expect { value }.to raise_error Restify::NetworkError, /without HTTP status/
+      end
+    end
+
+    # Unexpected errors must reject the promise too, as the caller would
+    # otherwise keep waiting for it until it times out.
+    context 'when building the response fails' do
+      def easy
+        super.tap do |double|
+          allow(double).to receive(:response_headers).and_raise('kaboom')
+        end
+      end
+
+      it 'rejects the promise' do
+        expect { value }.to raise_error 'kaboom'
+      end
+    end
+  end
+
+  describe 'waiting on requests' do
+    let(:adapter) { described_class.new }
+
+    # Threads that completed a transfer, i.e. ran the event loop.
+    let(:completed_by) { Queue.new }
+
+    before do
+      stub_request(:get, 'http://stubserver/fast')
+        .to_return(status: 200, body: '{}')
+      stub_request(:get, 'http://stubserver/slow')
+        .to_return { sleep 0.5 and {status: 200, body: '{}'} }
+
+      allow(adapter).to receive(:complete).and_wrap_original do |m, *args|
+        completed_by << Thread.current
+        m.call(*args)
+      end
+    end
+
+    def request(path)
+      Restify::Request.new(uri: "http://localhost:9292/#{path}")
+    end
+
+    def completed_by_threads
+      Array.new(completed_by.size) { completed_by.pop }
+    end
+
+    it 'runs the event loop in the waiting thread' do
+      expect(adapter.call(request('fast')).value!.code).to eq 200
+      expect(completed_by_threads).to eq [Thread.current]
+    end
+
+    it 'completes requests nobody waits on in the background' do
+      adapter.call(request('fast'))
+
+      # Poll without waiting on the promise:
+      Timeout.timeout(1) { sleep 0.01 while completed_by.empty? }
+
+      expect(completed_by_threads).not_to include Thread.current
+    end
+
+    it 'times out without blocking other requests' do
+      expect { adapter.call(request('slow')).value!(0.1) }.to raise_error Timeout::Error
+      expect(adapter.call(request('fast')).value!.code).to eq 200
+    end
+
+    it 'processes requests from many threads' do
+      threads = Array.new(8) do
+        Thread.new do
+          Array.new(5) { adapter.call(request('fast')).value!.code }
+        end
+      end
+
+      expect(threads.flat_map(&:value)).to all eq 200
+    end
+
+    # Only the adapter's own promises are completed by whichever thread
+    # runs the event loop. Callbacks chained to them must still run in
+    # the thread waiting on them, and only when waiting on them.
+    describe 'chained callbacks' do
+      it 'does not run them before waiting on them' do
+        ran_in = Queue.new
+        promise = adapter.call(request('fast'))
+        chained = promise.then { ran_in << Thread.current }
+
+        promise.value!
+        expect(ran_in).to be_empty
+
+        chained.value!
+        expect(ran_in.pop(timeout: 1)).to eq Thread.current
+      end
+
+      it 'runs them in the waiting thread when another thread runs the loop' do
+        # Keep the loop busy in another thread, until after the fast
+        # request completed.
+        driver = Thread.new { adapter.call(request('slow')).value! }
+        sleep 0.1
+
+        chained = adapter.call(request('fast')).then do |response|
+          [Thread.current, response.code]
+        end
+
+        expect(chained.value!).to eq [Thread.current, 200]
+        expect(completed_by_threads.first).to eq driver
+      ensure
+        driver&.join
+      end
+    end
+
+    context 'when waiting from within the event loop' do
+      it 'does not run the loop again' do
+        adapter.instance_variable_get(:@loop).acquire(0.1)
+
+        expect(adapter.drive(Restify::Promise.new, Restify::Timeout.new(0.1))).to be false
+      end
+    end
+  end
+
+  describe 'after fork' do
+    let(:adapter) { described_class.new }
+    let(:connects) { Queue.new }
+
+    before do
+      stub_request(:get, 'http://stubserver/fast')
+        .to_return(status: 200, body: '{}')
+      stub_request(:get, 'http://stubserver/slow')
+        .to_return { sleep 0.5 and {status: 200, body: '{}'} }
+
+      allow(adapter).to receive(:complete).and_wrap_original do |m, easy, *args|
+        connects << Ethon::Curl.get_info_long(:num_connects, easy.handle)
+        m.call(easy, *args)
+      end
+    end
+
+    def request(path)
+      Restify::Request.new(uri: "http://localhost:9292/#{path}")
+    end
+
+    def in_child
+      reader, writer = IO.pipe
+
+      pid = fork do
+        reader.close
+        result = begin
+          yield
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          e
+        end
+        writer.write(Marshal.dump(result))
+      ensure
+        # Skip at exit handlers of the parent
+        exit!(0)
+      end
+
+      writer.close
+      Marshal.load(reader.read) # rubocop:disable Security/MarshalLoad
+    ensure
+      reader&.close
+      Process.wait(pid) if pid
+    end
+
+    it 'does not use connections of the parent process' do
+      # The parent process pools and reuses its connection.
+      2.times { adapter.call(request('fast')).value! }
+      expect(Array.new(2) { connects.pop(timeout: 1) }).to eq [1, 0]
+
+      result = in_child do
+        code = adapter.call(request('fast')).value!.code
+        [code, connects.pop(timeout: 1)]
+      end
+
+      # A new connection, not the one pooled by the parent process
+      expect(result).to eq [200, 1]
+
+      # The parent process can still use its connections
+      expect(adapter.call(request('fast')).value!.code).to eq 200
+    end
+
+    it 'rejects requests started before the fork in the child process' do
+      promise = adapter.call(request('slow'))
+
+      result = in_child do
+        promise.value!
+      rescue Restify::NetworkError => e
+        e.message
+      end
+
+      expect(result).to include 'before fork'
+      expect(promise.value!.code).to eq 200
+    end
+  end
+end

data/spec/restify/adapter/loop_lock_spec.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Restify::Adapter::LoopLock do
+  subject(:lock) { described_class.new { interrupts << Thread.current } }
+
+  let(:interrupts) { Queue.new }
+  let(:threads) { [] }
+
+  after { threads.each(&:kill).each(&:join) }
+
+  # Run the block in another thread, and wait until it has returned.
+  # The thread is kept alive until the example finished.
+  def other(&)
+    done = Concurrent::Event.new
+    threads << Thread.new do
+      yield
+      done.set
+      sleep
+    end
+    expect(done.wait(1)).to be true
+  end
+
+  def elapsed
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    yield
+    Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+  end
+
+  describe '#acquire' do
+    it 'acquires a free loop' do
+      expect(lock.acquire(1)).to be true
+      expect(lock).to be_owned
+    end
+
+    it 'does not acquire the loop when done' do
+      expect(lock.acquire(1) { true }).to be false
+      expect(lock).not_to be_owned
+    end
+
+    it 'times out while another thread holds the loop' do
+      other { lock.acquire(1) }
+
+      expect(elapsed { expect(lock.acquire(0.1)).to be false }).to be >= 0.1
+      expect(lock).not_to be_owned
+    end
+
+    it 'waits for another thread to release the loop' do
+      acquired = Concurrent::Event.new
+      released = false
+      threads << Thread.new do
+        lock.acquire(1)
+        acquired.set
+        sleep 0.1
+        released = true
+        lock.release
+      end
+      expect(acquired.wait(1)).to be true
+
+      expect(lock.acquire(1)).to be true
+      expect(released).to be true
+    end
+
+    it 'stops waiting when done after being notified' do
+      done = false
+      other { lock.acquire(1) }
+      threads << Thread.new do
+        sleep 0.1
+        done = true
+        lock.notify
+      end
+
+      expect(elapsed { expect(lock.acquire(1) { done }).to be false }).to be < 0.5
+    end
+
+    it 'treats a dead thread as not holding the loop' do
+      Thread.new { lock.acquire(1) }.join
+
+      expect(lock.acquire(0.1)).to be true
+    end
+  end
+
+  describe '#release' do
+    it 'lets a waiting thread acquire the loop' do
+      lock.acquire(1)
+      waiter = Thread.new { lock.acquire(1) }
+      sleep 0.05
+
+      lock.release
+
+      expect(waiter.join(1)&.value).to be true
+      expect(lock).not_to be_owned
+    end
+  end
+
+  describe 'with a background thread' do
+    let(:background) { Queue.new }
+
+    # A background thread running the loop until interrupted, like an
+    # event loop waiting for sockets. Blocks on `interrupts` on purpose,
+    # and is killed after each example.
+    def start_background
+      threads << Thread.new do
+        loop do
+          lock.acquire_background
+          background << :acquired
+          interrupts.pop
+          lock.release
+        end
+      end
+
+      expect(background.pop(timeout: 1)).to eq :acquired
+    end
+
+    it 'interrupts the background thread to release the loop' do
+      start_background
+
+      expect(lock.acquire(1)).to be true
+      expect(background).to be_empty
+    end
+
+    it 'blocks the background thread while a thread holds the loop' do
+      start_background
+      lock.acquire(1)
+
+      sleep 0.1
+      expect(background).to be_empty
+
+      lock.release
+      expect(background.pop(timeout: 1)).to eq :acquired
+    end
+  end
+
+  describe '#reset!' do
+    it 'forgets threads holding the loop' do
+      other { lock.acquire(1) }
+
+      lock.reset!
+
+      expect(lock.acquire(0.1)).to be true
+    end
+  end
+end