thtp 0.1.0

data/lib/thtp/server/middleware.rb

@@ -0,0 +1,63 @@
+require 'thrift'
+require 'thtp/errors'
+
+module THTP
+  class Server
+    module Middleware
+      # Raise Thrift validation issues as their own detectable error type, rather
+      # than just ProtocolException.
+      class SchemaValidation
+        def initialize(app)
+          require 'thrift/validator' # if you don't have it, you'll need it
+          @app = app
+          @validator = Thrift::Validator.new
+        end
+
+        # Raises a ValidationError if any part of the request or response did not
+        # match the schema
+        def call(rpc, *rpc_args, **rpc_opts)
+          @validator.validate(rpc_args)
+          @app.call(rpc, *rpc_args, **rpc_opts).tap { |resp| @validator.validate(resp) }
+        rescue Thrift::ProtocolException => e
+          raise ServerValidationError, e.message
+        end
+      end
+
+      # Performs explicit rather than implicit AR connection management to ensure
+      # we don't run out of SQL connections. Note that this approach is
+      # suboptimal from a contention standpoint (better to check out once per
+      # thread), but that sync time should be irrelevant if we size our pool
+      # correctly, which we do. It is also suboptimal if we have any handler
+      # methods that do not hit the database at all, but that's unlikely.
+      #
+      # For more details, check out (get it?):
+      # https://bibwild.wordpress.com/2014/07/17/activerecord-concurrency-in-rails4-avoid-leaked-connections/
+      #
+      # This is probably only useful on servers.
+      class ActiveRecordPool
+        def initialize(app)
+          require 'active_record' # if you don't have it, why do you want this?
+          @app = app
+        end
+
+        def call(rpc, *rpc_args_and_opts)
+          ActiveRecord::Base.connection_pool.with_connection { @app.call(rpc, *rpc_args_and_opts) }
+        end
+      end
+
+      # Instruments RPCs for Skylight; requires Skylight to be initialised properly elsewhere
+      class Skylight
+        def initialize(app)
+          require 'skylight'
+          @app = app
+        end
+
+        def call(rpc, *rpc_args_and_opts)
+          ::Skylight.trace(rpc, 'rpc') do
+            @app.call(rpc, *rpc_args_and_opts)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/thtp/server/null_route.rb

@@ -0,0 +1,22 @@
+require 'rack'
+require 'thrift'
+
+require 'thtp/encoding'
+require 'thtp/errors'
+require 'thtp/status'
+require 'thtp/utils'
+
+module THTP
+  # A THTP middleware stack terminator, telling clients they've done
+  # something wrong if their requests weren't captured by a running server
+  class NullRoute
+    # if a request makes it here, it's bad; tell the caller what it should have done
+    def call(env)
+      # default to CompactProtocol because we need a protocol with which to send back errors
+      protocol = Encoding.protocol(Rack::Request.new(env).media_type) || Thrift::JsonProtocol
+      headers = { Rack::CONTENT_TYPE => Encoding.content_type(protocol) }
+      body = Utils.serialize_buffer(BadRequestError.new.to_thrift, protocol)
+      [Status::EXCEPTION, headers, [body]]
+    end
+  end
+end

data/lib/thtp/server/pub_sub.rb

@@ -0,0 +1,25 @@
+module THTP
+  class Server
+    # A truly trivial pub/sub implementation for instrumentation. Note, NOT
+    # threadsafe; make sure all subscribers are added before publishing anything.
+    module PubSub
+      # Add listeners to be run in the order of subscription
+      def subscribe(subscriber)
+        subscribers << subscriber
+      end
+
+      private
+
+      # If a subscriber raises an exception, any future ones won't run: this is
+      # not considered a bug. Don't raise.
+      def publish(event, *args)
+        # freeze to prevent any subscriber changes after usage
+        subscribers.freeze.each { |l| l.send(event, *args) if l.respond_to?(event) }
+      end
+
+      def subscribers
+        @subscribers ||= []
+      end
+    end
+  end
+end

data/lib/thtp/status.rb

@@ -0,0 +1,10 @@
+module THTP
+  # These HTTP status codes correspond to Thrift::MessageTypes, which tell
+  # clients what kind of message body to expect.
+  module Status
+    # CALL is not needed because HTTP has explicit requests
+    # ONEWAY is not supported in Ruby, but regardless, it has the same semantics as CALL
+    REPLY = 200
+    EXCEPTION = 500
+  end
+end

data/lib/thtp/utils.rb

@@ -0,0 +1,120 @@
+require 'thrift'
+
+require 'thtp/errors'
+
+module THTP
+  # Methods for interacting with generated Thrift files
+  module Utils
+    extend self
+
+    ### Timing
+
+    # get the current time, for benchmarking purposes. monotonic time is better
+    # than Time.now for this purposes. note, only works on Ruby 2.1.8+
+    def get_time
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # useful for benchmarking. returns wall clock ms since start time
+    def elapsed_ms(start_time)
+      ((get_time - start_time) * 1000).round
+    end
+
+    ### Thrift definition hacks
+
+    # @return array<Symbol> for a given thrift Service using reflection
+    #   because the Thrift compiler's generated definitions do not lend
+    #   themselves to external use
+    def extract_rpcs(thrift_service)
+      # it's really the Processor we want (the Client would work too)
+      root = thrift_service < Thrift::Processor ? thrift_service : thrift_service::Processor
+      # get all candidate classes that may contribute RPCs to this service
+      root.ancestors.flat_map do |klass|
+        next [] unless klass < Thrift::Processor
+        klass.instance_methods(false).
+          select { |method_name| method_name =~ /^process_/ }.
+          map { |method_name| method_name.to_s.sub(/^process_/, '').to_sym }
+      end
+    end
+
+    # args class is named after RPC, e.g., #get_things => Get_things_args
+    def args_class(service, rpc)
+      service.const_get("#{rpc.capitalize}_args")
+    end
+
+    # result class is named after RPC, e.g., #do_stuff => Do_stuff_result
+    def result_class(service, rpc)
+      service.const_get("#{rpc.capitalize}_result")
+    end
+
+    ### Thrift deserialisation
+
+    def deserialize(transport, struct, protocol)
+      struct.read(protocol.new(transport)) # read off the stream into Thrift objects
+      struct # return input object with all fields filled out
+    rescue Thrift::ProtocolException, EOFError => e
+      raise DeserializationError, e
+    end
+
+    def deserialize_buffer(buffer, struct, protocol)
+      deserialize(Thrift::MemoryBufferTransport.new(buffer), struct, protocol)
+    end
+
+    def deserialize_stream(in_stream, struct, protocol)
+      deserialize(Thrift::IOStreamTransport.new(in_stream, nil), struct, protocol)
+    end
+
+    # Thrift serialisation
+
+    def serialize(base_struct, transport, protocol)
+      base_struct.write(protocol.new(transport))
+      transport.tap(&:flush)
+    rescue Thrift::ProtocolException, EOFError => e
+      raise SerializationError, e
+    end
+
+    def serialize_buffer(base_struct, protocol)
+      transport = Thrift::MemoryBufferTransport.new
+      serialize(base_struct, transport, protocol) # write to output transport, an in-memory buffer
+      transport.read(transport.available) # return serialised thrift
+    end
+
+    def serialize_stream(base_struct, out_stream, protocol)
+      serialize(base_struct, Thrift::IOStreamTransport.new(nil, out_stream), protocol)
+    end
+
+    # String and object transforms
+
+    # Used to turn classes to canonical strings for routing and logging, e.g.,
+    # - MyServices::Thing::ThingService -> MyService.Thing.ThingService
+    # - MyServices::Thing::ThingRequest -> MyService.Thing.ThingRequest
+    # - MyServices::Thing::BadException -> MyService.Thing.BadException
+    # @param klass [Class]
+    def canonical_name(klass)
+      klass.name.gsub('::', '.')
+    end
+
+    # Recursive object serialiser compatible with anything likely to appear
+    # in Thrift, in case ActiveSupport's #as_json monkeypatch is unavailable
+    def jsonify(obj)
+      return obj.as_json if obj.respond_to?(:as_json) # ActiveSupport shortcut
+
+      case obj
+      when nil, false, true, Numeric # directly representable as JSON
+        obj
+      when Hash
+        obj.each_with_object({}) { |(k, v), h| h[jsonify(k)] = jsonify(v) }
+      when Enumerable
+        obj.map { |v| jsonify(v) }
+      else
+        if obj.instance_variables.any?
+          obj.instance_variables.each_with_object({}) do |iv, h|
+            h[iv.to_s[1..-1].to_sym] = jsonify(obj.instance_variable_get(iv))
+          end
+        else
+          obj.to_s # lowest common denominator serialisation
+        end
+      end
+    end
+  end
+end

data/lib/thtp/version.rb

@@ -0,0 +1,3 @@
+module THTP
+  VERSION = '0.1.0'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,36 @@
+require 'bundler/setup'
+require 'thtp'
+
+# test helpers
+require 'rack/test'
+require 'webmock'
+
+# require all spec helpers
+Dir[File.expand_path('support/**/*.rb', __dir__)].each do |f|
+  require f
+end
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = '.rspec_status'
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+
+  # Prevents mocking or stubbing a method that does not exist
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.warnings = true
+
+  config.profile_examples = 10
+
+  # randomise test order, allowing repeatable runs via --seed
+  config.order = :random
+  Kernel.srand config.seed
+end

data/spec/support/helpers/rack_test.rb

@@ -0,0 +1,3 @@
+RSpec.configure do |config|
+  config.include Rack::Test::Methods
+end

data/spec/support/helpers/thrift.rb

@@ -0,0 +1,163 @@
+module ThriftHelpers
+  include THTP::Utils
+
+  def read_struct(buffer, struct, protocol)
+    deserialize_buffer(buffer, struct, protocol)
+  end
+
+  def read_exception(buffer, protocol)
+    read_struct(buffer, Thrift::ApplicationException.new, protocol)
+  end
+
+  def read_result_response(response, service, rpc)
+    protocol = THTP::Encoding.protocol(response.media_type)
+    result_struct = THTP::Utils.result_class(service, rpc).new
+    read_struct(response.body, result_struct, protocol)
+  end
+
+  def read_exception_response(response)
+    protocol = THTP::Encoding.protocol(response.media_type)
+    read_exception(response.body, protocol)
+  end
+
+  def get_result_type(result_struct)
+    result_struct.struct_fields.each_value do |field|
+      return field[:name] unless result_struct.send(field[:name]).nil?
+    end
+  end
+
+  def get_result_response(response, service, rpc)
+    result = read_result_response(response, service, rpc)
+    field = get_result_type(result)
+    return result.send(field) if field
+    return nil unless result.respond_to?(:success)
+    raise THTP::BadResponseError, rpc
+  end
+end
+
+RSpec.configure do |config|
+  config.include ThriftHelpers
+end
+
+RSpec::Matchers.define :be_thrift_struct do |klass, attrs|
+  match do |struct|
+    expect(struct).to be_a klass
+    expect(struct).to have_attributes(attrs) unless attrs.nil?
+  end
+end
+
+RSpec::Matchers.define :be_thrift_result_response do |service, rpc|
+  match do |response|
+    return false if service.nil? || rpc.nil?
+    return false if response.status != THTP::Status::REPLY
+    read_result_response(response, service, rpc)
+    true
+  rescue THTP::SerializationError
+    false
+  end
+
+  failure_message do |response|
+    return 'service and rpc must be specified' if service.nil? || rpc.nil?
+
+    read_result_response(response, service, rpc)
+
+    if response.status != THTP::Status::REPLY
+      "expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
+    end
+  rescue THTP::SerializationError
+    'expected a Thrift ApplicationException, but deserialisation failed'
+  end
+end
+
+RSpec::Matchers.define :be_a_thrift_success_response do |service, rpc, klass, attrs|
+  match do |response|
+    return false if service.nil? || rpc.nil?
+    return false if response.status != THTP::Status::REPLY
+    result = read_result_response(response, service, rpc)
+    result_type = get_result_type(result)
+    return false unless result_type.nil? || result_type == 'success'
+    expect(result.success).to be_thrift_struct(klass, attrs) if klass && result_type
+    true
+  rescue THTP::SerializationError, Thrift::BadResponseError
+    false
+  end
+
+  failure_message do |response|
+    return 'service and rpc must be specified' if service.nil? || rpc.nil?
+
+    result = read_result_response(response, service, rpc)
+    result_type = get_result_type(result)
+
+    if response.status != THTP::Status::REPLY
+      "expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
+    elsif !result_type.nil? && result_type != 'success'
+      "expected successful Thrift result, but was #{result_type} instead"
+    elsif klass && result_type
+      expect(result.success).to be_thrift_struct(klass, attrs)
+    end
+  rescue THTP::SerializationError
+    'expected a Thrift ApplicationException, but deserialisation failed'
+  rescue Thrift::BadResponseError
+    'expected a result, but no result field was set'
+  end
+end
+
+RSpec::Matchers.define :be_a_thrift_exception_response do |service, rpc, klass, attrs|
+  match do |response|
+    return false if service.nil? || rpc.nil?
+    return false if response.status != THTP::Status::REPLY
+    result = read_result_response(response, service, rpc)
+    result_type = get_result_type(result)
+    return false if result_type.nil? || result_type == 'success'
+    expect(result.send(result_type)).to be_thrift_struct(klass, attrs) if klass
+    true
+  rescue THTP::SerializationError, Thrift::BadResponseError
+    false
+  end
+
+  failure_message do |response|
+    return 'service and rpc must be specified' if service.nil? || rpc.nil?
+
+    result = read_result_response(response, service, rpc)
+    result_type = get_result_type(result)
+
+    if response.status != THTP::Status::REPLY
+      "expected HTTP status #{THTP::Status::REPLY}, but was #{response.status}"
+    elsif result_type.nil? || result_type == 'success'
+      "expected Thrift exception, but was #{result_type} instead"
+    elsif klass
+      expect(result.send(result_type)).to be_thrift_struct(klass, attrs)
+    end
+  rescue THTP::SerializationError
+    'expected a Thrift ApplicationException, but deserialisation failed'
+  rescue Thrift::BadResponseError
+    'expected a result, but no result field was set'
+  end
+end
+
+RSpec::Matchers.define :be_thrift_error_response do |error_klass, message|
+  match do |response|
+    error = read_exception_response(response)
+
+    return false if response.status != THTP::Status::EXCEPTION
+    return false if !error_klass.nil? && error.type != error_klass.type
+    expect(error.message).to match(message) unless message.nil?
+    true
+  rescue THTP::SerializationError
+    false
+  end
+
+  failure_message do |response|
+    error = read_exception_response(response)
+
+    if response.status != THTP::Status::EXCEPTION
+      "expected HTTP status #{THTP::Status::EXCEPTION}, but was #{response.status}"
+    elsif !error_klass.nil? && error.type != error_klass.type
+      "expected ApplicationExceptionType #{error_klass.type}, but was #{error.type}"
+    elsif !message.nil?
+      expect(error.message).to match(message)
+    end
+  rescue THTP::SerializationError
+    'expected a Thrift ApplicationException, but deserialisation failed'
+  end
+end