thrifter 0.1.0

data/circle.yml

@@ -0,0 +1,10 @@
+machine:
+  services:
+    - docker
+dependencies:
+  override:
+    - echo 'Nothing to see here'
+test:
+  override:
+    - make test-ci
+    - make clean

data/lib/thrifter.rb

@@ -0,0 +1,166 @@
+require 'thrifter/version'
+
+require 'forwardable'
+require 'uri'
+require 'tnt'
+require 'concord'
+require 'thrift'
+require 'thrift-base64'
+require 'middleware'
+require 'connection_pool'
+
+module Thrifter
+  RPC = Struct.new(:name, :args)
+
+  class MiddlewareStack < Middleware::Builder
+    def finalize!
+      stack.freeze
+      to_app
+    end
+  end
+
+  class NullStatsd
+    def time(*)
+      yield
+    end
+
+    def increment(*)
+
+    end
+  end
+
+  RESERVED_METHODS = [
+    :send_message,
+    :send_oneway_message,
+    :send_message_args
+  ]
+
+  Configuration = Struct.new :transport, :protocol,
+    :pool_size, :pool_timeout,
+    :uri, :rpc_timeout,
+    :stack, :statsd
+
+  class << self
+    def build(client_class, &block)
+      rpcs = client_class.instance_methods.each_with_object([ ]) do |method_name, rpcs|
+        next if RESERVED_METHODS.include? method_name
+        next unless method_name =~ /^send_(?<rpc>.+)$/
+        rpcs << Regexp.last_match[:rpc].to_sym
+      end
+
+      rpcs.freeze
+
+      Class.new Client do
+        rpcs.each do |rpc_name|
+          define_method rpc_name do |*args|
+            invoke RPC.new(rpc_name, args)
+          end
+        end
+
+        class_eval(&block) if block
+
+        private
+
+        define_method :rpcs do
+          rpcs
+        end
+
+        define_method :client_class do
+          client_class
+        end
+      end
+    end
+  end
+
+  class Client
+    class Dispatcher
+      include Concord.new(:app, :transport, :client)
+
+      def call(rpc)
+        transport.open
+        client.send rpc.name, *rpc.args
+      ensure
+        transport.close
+      end
+    end
+
+    class << self
+      extend Forwardable
+
+      attr_accessor :config
+
+      def_delegators :config, :stack
+      def_delegators :stack, :use
+
+      def configure
+        yield config
+      end
+
+      # NOTE: the inherited hook is better than doing singleton
+      # methods for config. This works when Thrifter is used like a
+      # struct MyClient = Thrifter.build(MyService) or like delegate
+      # class MyClient < Thrifter.build(MyService). The end result is
+      # each class has it's own configuration instance.
+      def inherited(base)
+        base.config = Configuration.new
+        base.configure do |config|
+          config.transport = Thrift::FramedTransport
+          config.protocol = Thrift::BinaryProtocol
+          config.pool_size = 12
+          config.pool_timeout = 0.1
+          config.rpc_timeout = 0.3
+          config.statsd = NullStatsd.new
+          config.stack = MiddlewareStack.new
+        end
+      end
+    end
+
+    def initialize
+      fail ArgumentError, 'config.uri not set!' unless config.uri
+
+      uri = URI(config.uri)
+
+      fail ArgumentError, 'URI did not contain port' unless uri.port
+
+      @pool = ConnectionPool.new size: config.pool_size.to_i, timeout: config.pool_timeout.to_f do
+        stack = MiddlewareStack.new
+
+        stack.use config.stack
+
+        # Insert metrics here so metrics are as close to the network
+        # as possible. This excludes time in any middleware an
+        # application may have configured.
+        stack.use StatsdMiddleware, config.statsd
+
+        socket = Thrift::Socket.new uri.host, uri.port, config.rpc_timeout.to_f
+        transport = config.transport.new socket
+        protocol = config.protocol.new transport
+
+        stack.use Dispatcher, transport, client_class.new(protocol)
+
+        stack.finalize!
+      end
+    end
+
+    private
+
+    def pool
+      @pool
+    end
+
+    def config
+      self.class.config
+    end
+
+    def invoke(rpc)
+      pool.with do |stack|
+        stack.call rpc
+      end
+    end
+  end
+end
+
+require_relative 'thrifter/statsd_middleware'
+require_relative 'thrifter/ping'
+require_relative 'thrifter/error_wrapping_middleware'
+require_relative 'thrifter/retry'

data/lib/thrifter/error_wrapping_middleware.rb

@@ -0,0 +1,49 @@
+module Thrifter
+  ClientError = Tnt.boom do |ex|
+    "#{ex.class}: #{ex.message}"
+  end
+
+  class ErrorWrappingMiddleware
+    WRAP = [
+      Thrift::TransportException,
+      Thrift::ProtocolException,
+      Thrift::ApplicationException,
+      Timeout::Error,
+
+      # This exception is a superclass for all Errno things coming
+      # from the operating system network stack. See the documentation
+      # on Error no for more information.
+      SystemCallError
+    ]
+
+    class << self
+      def wrapped
+        WRAP
+      end
+    end
+
+    def initialize(app, extras = [ ])
+      @app, @extras = app, extras
+    end
+
+    def call(rpc)
+      app.call rpc
+    rescue *wrapped => ex
+      raise ClientError, ex
+    end
+
+    private
+
+    def app
+      @app
+    end
+
+    def extras
+      @extras
+    end
+
+    def wrapped
+      self.class.wrapped + extras
+    end
+  end
+end

data/lib/thrifter/ping.rb

@@ -0,0 +1,10 @@
+module Thrifter
+  module Ping
+    def up?
+      ping
+      true
+    rescue
+      false
+    end
+  end
+end

data/lib/thrifter/queueing.rb

@@ -0,0 +1,38 @@
+require 'sidekiq'
+require 'sidekiq-thrift_arguments'
+
+module Thrifter
+  module Queueing
+    class Job
+      include Sidekiq::Worker
+      include Sidekiq::ThriftArguments
+
+      # NOTE: sidekiq-thrift_arguments does not recurse into
+      # arguments. The thrift objects must not be inside an array or
+      # other structure. This is why the method has so many arguments.
+      # Sidekik-thrift_arguments will correctly pick up any complex
+      # type in the splat and handle serialization/deserialization
+      def perform(klass, rpc_name, *rpc_args)
+        client = klass.constantize.new
+        client.send rpc_name, *rpc_args
+      end
+    end
+
+    class Proxy
+      def initialize(klass, rpcs)
+        rpcs.each do |name|
+          define_singleton_method name do |*args|
+            job_args = [ klass.to_s, name ].concat(args)
+            Job.perform_async(*job_args)
+          end
+        end
+      end
+    end
+
+    def queued
+      Proxy.new(self.class, rpcs).tap do |proxy|
+        yield proxy if block_given?
+      end
+    end
+  end
+end

data/lib/thrifter/retry.rb

@@ -0,0 +1,57 @@
+module Thrifter
+  RetryError = Tnt.boom do |count, rpc|
+    "#{rpc} RPC unsuccessful after #{count} times"
+  end
+
+  module Retry
+    RETRIABLE_ERRORS = [
+      Thrift::TransportException,
+      Thrift::ProtocolException,
+      Thrift::ApplicationException,
+      Timeout::Error,
+      Errno::ECONNREFUSED,
+      Errno::EADDRNOTAVAIL,
+      Errno::EHOSTUNREACH,
+      Errno::EHOSTDOWN,
+      Errno::ETIMEDOUT
+    ]
+
+    class Proxy
+      attr_reader :tries, :interval, :client
+
+      def initialize(client, tries, interval, rpcs)
+        @client, @tries, @interval = client, tries, interval
+
+        rpcs.each do |name|
+          define_singleton_method name do |*args|
+            invoke_with_retry(name, *args)
+          end
+        end
+      end
+
+      private
+
+      def invoke_with_retry(name, *args)
+        counter = 0
+
+        begin
+          counter = counter + 1
+          client.send name, *args
+        rescue *RETRIABLE_ERRORS => ex
+          if counter < tries
+            sleep interval
+            retry
+          else
+            raise RetryError.new(tries, name)
+          end
+        end
+      end
+    end
+
+    def with_retry(tries: 5, interval: 0.01)
+      Proxy.new(self, tries, interval, rpcs).tap do |proxy|
+        yield proxy if block_given?
+      end
+    end
+  end
+end

data/lib/thrifter/statsd_middleware.rb

@@ -0,0 +1,26 @@
+module Thrifter
+  class StatsdMiddleware
+    include Concord.new(:app, :statsd)
+
+    def call(rpc)
+      statsd.time rpc.name do
+        app.call rpc
+      end
+    rescue Thrift::TransportException => ex
+      statsd.increment 'errors.transport'
+      raise ex
+    rescue Thrift::ProtocolException => ex
+      statsd.increment 'errors.protocol'
+      raise ex
+    rescue Thrift::ApplicationException => ex
+      statsd.increment 'errors.application'
+      raise ex
+    rescue Timeout::Error => ex
+      statsd.increment 'errors.timeout'
+      raise ex
+    rescue => ex
+      statsd.increment 'errors.other'
+      raise ex
+    end
+  end
+end

data/lib/thrifter/version.rb

@@ -0,0 +1,3 @@
+module Thrifter
+  VERSION = "0.1.0"
+end

data/script/monkey-client

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+$stdout.sync = true
+$stderr.sync = true
+
+uri = ARGV[0]
+
+if uri.nil?
+  abort 'USAGE: monkey-client HOST:PORT'
+end
+
+require 'bundler/setup'
+require 'thrifter'
+require 'thrifter/queueing'
+require 'securerandom'
+
+require 'eventmachine'
+
+root = File.expand_path '../..', __FILE__
+$LOAD_PATH << "#{root}/vendor/gen-rb"
+
+require 'test_service'
+
+class MonkeyClient < Thrifter.build(TestService::Client)
+  include Thrifter::Retry
+
+  config.uri = "tcp://#{ARGV[0]}"
+end
+
+client = MonkeyClient.new
+
+EM.run do
+  rand(1..5).times do
+    EM.add_periodic_timer rand(0.1..2.0) do
+      client.echo(TestMessage.new(message: "Echo #{SecureRandom.hex(16)}"))
+    end
+  end
+
+  rand(1..5).times do
+    EM.add_periodic_timer rand(0.1..2.0) do
+      client.onewayEcho(TestMessage.new(message: "Oneway: #{SecureRandom.hex(16)}"))
+    end
+  end
+
+  rand(1..5).times do
+    EM.add_periodic_timer rand(0.1..2.0) do
+      client.with_retry.echo(TestMessage.new(message: 'retried echo'))
+    end
+  end
+
+  rand(1..5).times do
+    EM.add_periodic_timer rand(0.1..2.0) do
+      client.with_retry.onewayEcho(TestMessage.new(message: 'retried oneway echo'))
+    end
+  end
+
+  EM.add_timer 30 do
+    puts "Good monkey...here's your banana."
+    EM.stop
+  end
+end

data/script/server

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+$stdout.sync = true
+$stderr.sync = true
+
+require 'bundler/setup'
+require 'thrift'
+
+root = File.expand_path '../..', __FILE__
+$LOAD_PATH << "#{root}/vendor/gen-rb"
+
+require 'test_service'
+
+puts 'Starting on port 9090.....'
+
+class Handler
+  def echo(message)
+    puts "echo --#{message.inspect}"
+    message
+  end
+
+  def onewayEcho(message)
+    puts "onewayEcho -- #{message.inspect}"
+    # nada
+  end
+end
+
+processor = TestService::Processor.new Handler.new
+transport = Thrift::ServerSocket.new 9090
+transport_factory = Thrift::FramedTransportFactory.new
+server = Thrift::ThreadPoolServer.new processor, transport, transport_factory, nil, threads = 10
+server.serve