protobuffy 3.1.0

data/lib/protobuf/rpc/rpc.pb.rb

@@ -0,0 +1,53 @@
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+require 'protobuf/message'
+
+module Protobuf
+  module Socketrpc
+
+    ##
+    # Enum Classes
+    #
+    class ErrorReason < ::Protobuf::Enum
+      define :BAD_REQUEST_DATA, 0
+      define :BAD_REQUEST_PROTO, 1
+      define :SERVICE_NOT_FOUND, 2
+      define :METHOD_NOT_FOUND, 3
+      define :RPC_ERROR, 4
+      define :RPC_FAILED, 5
+      define :INVALID_REQUEST_PROTO, 6
+      define :BAD_RESPONSE_PROTO, 7
+      define :UNKNOWN_HOST, 8
+      define :IO_ERROR, 9
+    end
+
+
+    ##
+    # Message Classes
+    #
+    class Request < ::Protobuf::Message; end
+    class Response < ::Protobuf::Message; end
+
+
+    ##
+    # Message Fields
+    #
+    class Request
+      required :string, :service_name, 1
+      required :string, :method_name, 2
+      optional :bytes, :request_proto, 3
+      optional :string, :caller, 4
+    end
+
+    class Response
+      optional :bytes, :response_proto, 1
+      optional :string, :error, 2
+      optional :bool, :callback, 3, :default => false
+      optional ::Protobuf::Socketrpc::ErrorReason, :error_reason, 4
+    end
+
+  end
+
+end
+

data/lib/protobuf/rpc/server.rb

@@ -0,0 +1,39 @@
+require 'protobuf'
+require 'protobuf/logger'
+require 'protobuf/rpc/rpc.pb'
+require 'protobuf/rpc/buffer'
+require 'protobuf/rpc/env'
+require 'protobuf/rpc/error'
+require 'protobuf/rpc/middleware'
+require 'protobuf/rpc/service_dispatcher'
+
+module Protobuf
+  module Rpc
+    module Server
+      def gc_pause
+        ::GC.disable if ::Protobuf.gc_pause_server_request?
+
+        yield
+
+        ::GC.enable if ::Protobuf.gc_pause_server_request?
+      end
+
+      # Invoke the service method dictated by the proto wrapper request object
+      #
+      def handle_request(request_data)
+        # Create an env object that holds different parts of the environment and
+        # is available to all of the middlewares
+        env = Env.new('encoded_request' => request_data, 'log_signature' => log_signature)
+
+        # Invoke the middleware stack, the last of which is the service dispatcher
+        env = Rpc.middleware.call(env)
+
+        env.encoded_response
+      end
+
+      def log_signature
+        @_log_signature ||= "[server-#{self.class.name}]"
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/http/server.rb

@@ -0,0 +1,101 @@
+require 'protobuf/rpc/server'
+
+require 'rack'
+require 'rack/server'
+
+module Protobuf
+  module Rpc
+    module Http
+      class Server
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        # TODO: more comprehensive mapping?
+        HTTP_STATUSES = {
+          Protobuf::Socketrpc::ErrorReason::BAD_REQUEST_DATA => 400,
+          Protobuf::Socketrpc::ErrorReason::BAD_REQUEST_PROTO => 400,
+          Protobuf::Socketrpc::ErrorReason::SERVICE_NOT_FOUND => 404,
+          Protobuf::Socketrpc::ErrorReason::METHOD_NOT_FOUND => 404,
+          Protobuf::Socketrpc::ErrorReason::RPC_ERROR => 500,
+          Protobuf::Socketrpc::ErrorReason::RPC_FAILED => 500,
+          Protobuf::Socketrpc::ErrorReason::INVALID_REQUEST_PROTO => 500,
+          Protobuf::Socketrpc::ErrorReason::BAD_RESPONSE_PROTO => 500,
+          Protobuf::Socketrpc::ErrorReason::UNKNOWN_HOST => 500,
+          Protobuf::Socketrpc::ErrorReason::IO_ERROR => 500
+        }
+
+        def initialize(options = {})
+          @options = options
+        end
+
+        def log_signature
+          @_log_signature ||= "[http-server-#{self.class.name}]"
+        end
+
+        def call(env)
+          path_components = env['PATH_INFO'].split("/").map{ |x| CGI::unescape(x) }.compact.reject{ |x| x.empty? }
+          if path_components.length < 2
+            return protobuf_http_response 400,
+              :error => "Expected path format /CLASS/METHOD",
+              :reason => Protobuf::Socketrpc::ErrorReason::INVALID_REQUEST_PROTO
+          end
+
+          service_name = path_components[0..-2].join("::")
+          method_name = path_components[-1]
+
+          rpc_request = ::Protobuf::Socketrpc::Request.new(
+            :service_name => service_name,
+            :method_name => method_name,
+            :request_proto => env['rack.input'].read,
+            :caller => env['HTTP_X_PROTOBUF_CALLER'] || ''
+          )
+
+          encoded_request = rpc_request.encode()
+
+          begin
+            encoded_response = handle_request(encoded_request)
+          rescue Exception => e
+            return protobuf_http_response 500,
+              :error => "Handle request failed: #{e.to_s}",
+              :reason => Protobuf::Socketrpc::ErrorReason::RPC_ERROR
+          end
+
+          rpc_response = Protobuf::Socketrpc::Response.decode(encoded_response)
+
+          if rpc_response[:error].length > 0
+            status = HTTP_STATUSES[rpc_response[:error_reason]] or 500
+            return protobuf_http_response status,
+              :error => rpc_response[:error],
+              :reason => rpc_response[:error_reason]
+          end
+          
+          return protobuf_http_response 200, :body => rpc_response['response_proto']
+        end
+
+        def protobuf_http_response(status, options)
+          response = [status, { 'Content-Type' => 'application/x-protobuf' }, []]
+          response[1]['X-Protobuf-Error'] = options[:error] unless options[:error].nil?
+          response[1]['X-Protobuf-Error-Reason'] = options[:reason].to_s unless options[:reason].nil?
+          response[2] = [options[:body]] unless options[:body].nil?
+          response
+        end
+
+        def run
+          Rack::Server.start(
+            :app => self,
+            :Host => @options[:host],
+            :Port => @options[:port]
+          )
+          @running = true
+        end
+
+        def running?
+          !!@running
+        end
+
+        def stop
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/http_runner.rb

@@ -0,0 +1,34 @@
+module Protobuf
+  module Rpc
+    class HttpRunner
+
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.is_a?(Hash) then
+                     options
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     raise "Cannot parse HTTP Server - server options"
+                   end
+
+        @server = ::Protobuf::Rpc::Http::Server.new(@options)
+      end
+
+      def run
+        yield if block_given?
+        @server.run
+      end
+
+      def running?
+        @server.running?
+      end
+
+      def stop
+        @server.stop
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket/server.rb

@@ -0,0 +1,113 @@
+require 'protobuf/rpc/server'
+require 'protobuf/rpc/servers/socket/worker'
+
+module Protobuf
+  module Rpc
+    module Socket
+      class Server
+        include ::Protobuf::Logger::LogMethods
+
+        AUTO_COLLECT_TIMEOUT = 5 # seconds
+
+        def initialize(options)
+          @options = options
+        end
+
+        def cleanup?
+          # every 10 connections run a cleanup routine after closing the response
+          @threads.size > (@threshold - 1) && (@threads.size % @threshold) == 0
+        end
+
+        def cleanup_threads
+          log_debug { sign_message("Thread cleanup - #{@threads.size} - start") }
+
+          @threads = @threads.select do |t|
+            if t[:thread].alive?
+              true
+            else
+              t[:thread].join
+              @working.delete(t[:socket])
+              false
+            end
+          end
+
+          log_debug { sign_message("Thread cleanup - #{@threads.size} - complete") }
+        end
+
+        def log_signature
+          @_log_signature ||= "server-#{self.class.name}"
+        end
+
+        def new_worker(socket)
+          Thread.new(socket) do |sock|
+            ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
+              s.close
+            end
+          end
+        end
+
+        def run
+          log_debug { sign_message("Run") }
+          host = @options[:host]
+          port = @options[:port]
+          backlog = @options[:backlog]
+          @threshold = @options[:threshold]
+
+          @threads = []
+          @server = ::TCPServer.new(host, port)
+          raise "The server was unable to start properly." if @server.closed?
+
+          @server.listen(backlog)
+          @working = []
+          @listen_fds = [@server]
+          @running = true
+
+          while running?
+            log_debug { sign_message("Waiting for connections") }
+            ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT) rescue nil
+
+            if ready_cnxns
+              cnxns = ready_cnxns.first
+              cnxns.each do |client|
+                case
+                when !running? then
+                  # no-op
+                when client == @server then
+                  log_debug { sign_message("Accepted new connection") }
+                  client, sockaddr = @server.accept
+                  @listen_fds << client
+                else
+                  unless @working.include?(client)
+                    @working << @listen_fds.delete(client)
+                    log_debug { sign_message("Working")  }
+                    @threads << { :thread => new_worker(client), :socket => client }
+
+                    cleanup_threads if cleanup?
+                  end
+                end
+              end
+            else
+              # Run a cleanup if select times out while waiting
+              cleanup_threads if @threads.size > 1
+            end
+          end
+
+        rescue Errno::EADDRINUSE
+          raise
+        rescue
+          # Closing the server causes the loop to raise an exception here
+          raise #if running?
+        end
+
+        def running?
+          !!@running
+        end
+
+        def stop
+          @running = false
+          @server.try(:close)
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket/worker.rb

@@ -0,0 +1,56 @@
+require 'protobuf/rpc/server'
+require 'protobuf/logger'
+
+module Protobuf
+  module Rpc
+    module Socket
+      class Worker
+        include ::Protobuf::Rpc::Server
+        include ::Protobuf::Logger::LogMethods
+
+        def initialize(sock, &complete_cb)
+          @socket = sock
+          @complete_cb = complete_cb
+
+          data = read_data
+          return unless data
+
+          gc_pause do
+            encoded_response = handle_request(data)
+            send_data(encoded_response)
+          end
+        end
+
+        def read_data
+          size_io = StringIO.new
+
+          until (size_reader = @socket.getc) == "-"
+            size_io << size_reader
+          end
+          str_size_io = size_io.string
+
+          @socket.read(str_size_io.to_i)
+        end
+
+        def send_data(data)
+          raise 'Socket closed unexpectedly' unless socket_writable?
+          response_buffer = Protobuf::Rpc::Buffer.new(:write)
+          response_buffer.set_data(data)
+
+          @socket.write(response_buffer.write)
+          @socket.flush
+
+          @complete_cb.call(@socket)
+        end
+
+        def log_signature
+          @_log_signature ||= "server-#{self.class}-#{object_id}"
+        end
+
+        def socket_writable?
+          ! @socket.nil? && ! @socket.closed?
+        end
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/socket_runner.rb

@@ -0,0 +1,34 @@
+module Protobuf
+  module Rpc
+    class SocketRunner
+
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.is_a?(Hash) then
+                     options
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     raise "Cannot parser Socket Server - server options"
+                   end
+
+        @server = ::Protobuf::Rpc::Socket::Server.new(@options)
+      end
+
+      def run
+        yield if block_given?
+        @server.run
+      end
+
+      def running?
+        @server.running?
+      end
+
+      def stop
+        @server.stop
+      end
+    end
+  end
+end

data/lib/protobuf/rpc/servers/zmq/broker.rb

@@ -0,0 +1,155 @@
+require 'thread'
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Broker
+        include ::Protobuf::Rpc::Zmq::Util
+
+        attr_reader :local_queue
+
+        def initialize(server)
+          @server = server
+
+          init_zmq_context
+          init_local_queue
+          init_backend_socket
+          init_frontend_socket
+          init_poller
+        rescue
+          teardown
+          raise
+        end
+
+        def run
+          @idle_workers = []
+
+          loop do
+            unless local_queue.empty?
+              process_local_queue
+            end
+
+            rc = @poller.poll(500)
+
+            # The server was shutdown and no requests are pending
+            break if rc == 0 && !running?
+
+            # Something went wrong
+            break if rc == -1
+
+            @poller.readables.each do |readable|
+              case readable
+              when @frontend_socket
+                process_frontend
+              when @backend_socket
+                process_backend
+              end
+            end
+          end
+        ensure
+          teardown
+        end
+
+        def running?
+          @server.running? || @server.workers.any?
+        end
+
+        private
+
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket(ZMQ::ROUTER)
+          zmq_error_check(@backend_socket.bind(@server.backend_uri))
+        end
+
+        def init_frontend_socket
+          @frontend_socket = @zmq_context.socket(ZMQ::ROUTER)
+          zmq_error_check(@frontend_socket.bind(@server.frontend_uri))
+        end
+
+        def init_local_queue
+          @local_queue = ::Queue.new
+        end
+
+        def init_poller
+          @poller = ZMQ::Poller.new
+          @poller.register_readable(@frontend_socket)
+          @poller.register_readable(@backend_socket)
+        end
+
+        def init_zmq_context
+          if inproc?
+            @zmq_context = @server.zmq_context
+          else
+            @zmq_context = ZMQ::Context.new
+          end
+        end
+
+        def inproc?
+          !!@server.try(:inproc?)
+        end
+
+        def process_backend
+          worker, ignore, *frames = read_from_backend
+
+          @idle_workers << worker
+
+          unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
+            write_to_frontend(frames)
+          end
+        end
+
+        def process_frontend
+          address, _, message, *frames = read_from_frontend
+
+          if message == ::Protobuf::Rpc::Zmq::CHECK_AVAILABLE_MESSAGE
+            if @idle_workers.any? || local_queue.empty?
+              write_to_frontend([address, "", ::Protobuf::Rpc::Zmq::WORKERS_AVAILABLE])
+            else
+              write_to_frontend([address, "", ::Protobuf::Rpc::Zmq::NO_WORKERS_AVAILABLE])
+            end
+          else
+            if @idle_workers.any?
+              write_to_backend([@idle_workers.shift, ""] + [address, "", message ] + frames)
+            else
+              local_queue.push([address, "", message ] + frames)
+            end
+          end
+        end
+
+        def process_local_queue
+          return if local_queue.empty?
+          return if @idle_workers.empty?
+
+          write_to_backend([@idle_workers.shift, ""] + local_queue.pop)
+          process_local_queue
+        end
+
+        def read_from_backend
+          frames = []
+          zmq_error_check(@backend_socket.recv_strings(frames))
+          frames
+        end
+
+        def read_from_frontend
+          frames = []
+          zmq_error_check(@frontend_socket.recv_strings(frames))
+          frames
+        end
+
+        def teardown
+          @frontend_socket.try(:close)
+          @backend_socket.try(:close)
+          @zmq_context.try(:terminate)
+        end
+
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+
+        def write_to_frontend(frames)
+          zmq_error_check(@frontend_socket.send_strings(frames))
+        end
+      end
+    end
+  end
+end