protobuf 1.1.3 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/.gitignore +3 -0
  2. data/Gemfile.lock +44 -25
  3. data/README.md +2 -2
  4. data/Rakefile +15 -0
  5. data/bin/rpc_server +52 -11
  6. data/lib/protobuf.rb +22 -10
  7. data/lib/protobuf/common/logger.rb +26 -25
  8. data/lib/protobuf/descriptor/file_descriptor.rb +1 -1
  9. data/lib/protobuf/message/field.rb +2 -2
  10. data/lib/protobuf/rpc/buffer.rb +30 -25
  11. data/lib/protobuf/rpc/client.rb +8 -8
  12. data/lib/protobuf/rpc/connector.rb +2 -0
  13. data/lib/protobuf/rpc/connectors/base.rb +0 -1
  14. data/lib/protobuf/rpc/connectors/common.rb +48 -48
  15. data/lib/protobuf/rpc/connectors/em_client.rb +53 -27
  16. data/lib/protobuf/rpc/connectors/eventmachine.rb +14 -17
  17. data/lib/protobuf/rpc/connectors/socket.rb +23 -16
  18. data/lib/protobuf/rpc/connectors/zmq.rb +73 -0
  19. data/lib/protobuf/rpc/error.rb +1 -2
  20. data/lib/protobuf/rpc/error/client_error.rb +4 -4
  21. data/lib/protobuf/rpc/server.rb +31 -43
  22. data/lib/protobuf/rpc/servers/evented/server.rb +43 -0
  23. data/lib/protobuf/rpc/servers/evented_runner.rb +1 -1
  24. data/lib/protobuf/rpc/servers/socket/server.rb +108 -0
  25. data/lib/protobuf/rpc/servers/socket/worker.rb +59 -0
  26. data/lib/protobuf/rpc/servers/socket_runner.rb +3 -3
  27. data/lib/protobuf/rpc/servers/zmq/broker.rb +85 -0
  28. data/lib/protobuf/rpc/servers/zmq/server.rb +50 -0
  29. data/lib/protobuf/rpc/servers/zmq/util.rb +27 -0
  30. data/lib/protobuf/rpc/servers/zmq/worker.rb +72 -0
  31. data/lib/protobuf/rpc/servers/zmq_runner.rb +26 -0
  32. data/lib/protobuf/rpc/service.rb +5 -5
  33. data/lib/protobuf/version.rb +1 -1
  34. data/protobuf.gemspec +12 -10
  35. data/spec/benchmark/tasks.rb +37 -5
  36. data/spec/functional/evented_server_spec.rb +64 -0
  37. data/spec/functional/socket_server_spec.rb +63 -0
  38. data/spec/functional/zmq_server_spec.rb +63 -0
  39. data/spec/helper/server.rb +32 -12
  40. data/spec/lib/protobuf/message/encoder_spec.rb +19 -0
  41. data/spec/proto/test.pb.rb +3 -3
  42. data/spec/proto/test.proto +3 -3
  43. data/spec/proto/test_service.rb +1 -0
  44. data/spec/spec_helper.rb +6 -0
  45. data/spec/unit/message_spec.rb +1 -1
  46. data/spec/unit/rpc/client_spec.rb +11 -3
  47. data/spec/unit/rpc/connectors/common_spec.rb +0 -1
  48. data/spec/unit/rpc/connectors/eventmachine_client_spec.rb +32 -0
  49. data/spec/unit/rpc/connectors/socket_spec.rb +2 -4
  50. data/spec/unit/rpc/connectors/zmq_spec.rb +27 -0
  51. data/spec/unit/rpc/servers/evented_server_spec.rb +3 -3
  52. data/spec/unit/rpc/servers/socket_server_spec.rb +14 -13
  53. data/spec/unit/rpc/servers/zmq/broker_spec.rb +27 -0
  54. data/spec/unit/rpc/servers/zmq/server_spec.rb +37 -0
  55. data/spec/unit/rpc/servers/zmq/util_spec.rb +41 -0
  56. data/spec/unit/rpc/servers/zmq/worker_spec.rb +36 -0
  57. data/spec/unit/rpc/service_spec.rb +22 -18
  58. metadata +87 -40
  59. data/lib/protobuf/rpc/servers/evented_server.rb +0 -28
  60. data/lib/protobuf/rpc/servers/socket_server.rb +0 -146
@@ -0,0 +1,73 @@
1
+ require 'protobuf/rpc/connectors/base'
2
+
3
+ module Protobuf
4
+ module Rpc
5
+ module Connectors
6
+ class Zmq < Base
7
+ include Protobuf::Rpc::Connectors::Common
8
+ include Protobuf::Logger::LogMethods
9
+
10
+ def send_request
11
+ check_async
12
+ setup_connection
13
+ connect_to_rpc_server
14
+ post_init
15
+ read_response
16
+ ensure
17
+ @socket.close if @socket
18
+ @zmq_context.terminate if @zmq_context
19
+ @zmq_context = nil
20
+ end
21
+
22
+ private
23
+
24
+ def check_async
25
+ if async?
26
+ log_error { "[client-#{self.class}] Cannot run in async mode" }
27
+ raise "Cannot use Zmq client in async mode"
28
+ end
29
+ end
30
+
31
+ def close_connection
32
+ return if(@error)
33
+ zmq_error_check(@socket.close)
34
+ zmq_error_check(@zmq_context.terminate)
35
+ log_debug { "[client-#{self.class}] Connector closed" }
36
+ end
37
+
38
+ def connect_to_rpc_server
39
+ return if(@error)
40
+ log_debug { "[client-#{self.class} Establishing connection: #{options[:host]}:#{options[:port]}" }
41
+ @zmq_context = ::ZMQ::Context.new
42
+ @socket = @zmq_context.socket(::ZMQ::REQ)
43
+ zmq_error_check(@socket.connect("tcp://#{options[:host]}:#{options[:port]}"))
44
+ log_debug { "[client-#{self.class}] Connection established #{options[:host]}:#{options[:port]}" }
45
+ end
46
+
47
+ # Method to determine error state, must be used with Connector api
48
+ def error?
49
+ !!@error
50
+ end
51
+
52
+ def read_response
53
+ return if(@error)
54
+ @response_data = ''
55
+ zmq_error_check(@socket.recv_string(@response_data))
56
+ parse_response
57
+ end
58
+
59
+ def send_data
60
+ return if(@error)
61
+ log_debug { "[#{log_signature}] Sending Request: %s" % @request_data }
62
+ @stats.request_size = @request_data.size
63
+ zmq_error_check(@socket.send_string(@request_data))
64
+ log_debug { "[client-#{self.class}] write closed" }
65
+ end
66
+
67
+ def zmq_error_check(return_code)
68
+ raise "Last API call failed at #{caller(1)}" unless return_code >= 0
69
+ end
70
+ end
71
+ end
72
+ end
73
+ end
@@ -9,7 +9,7 @@ module Protobuf
9
9
  attr_reader :error_type
10
10
 
11
11
  def initialize message='An unknown RpcError occurred', error_type='RPC_ERROR'
12
- @error_type = error_type.is_a?(String) ? Protobuf::Socketrpc::ErrorReason.const_get(error_type) : error_type
12
+ @error_type = error_type.is_a?(String) ? ::Protobuf::Socketrpc::ErrorReason.const_get(error_type) : error_type
13
13
  super message
14
14
  end
15
15
 
@@ -18,7 +18,6 @@ module Protobuf
18
18
  response.error_reason = @error_type
19
19
  end
20
20
  end
21
-
22
21
  end
23
22
  end
24
23
 
@@ -4,25 +4,25 @@ module Protobuf
4
4
  module Rpc
5
5
 
6
6
  class InvalidRequestProto < PbError
7
- def initialize message='Invalid request type given'
7
+ def initialize(message='Invalid request type given')
8
8
  super message, 'INVALID_REQUEST_PROTO'
9
9
  end
10
10
  end
11
11
 
12
12
  class BadResponseProto < PbError
13
- def initialize message='Bad response type from server'
13
+ def initialize(message='Bad response type from server')
14
14
  super message, 'BAD_RESPONSE_PROTO'
15
15
  end
16
16
  end
17
17
 
18
18
  class UnkownHost < PbError
19
- def initialize message='Unknown host or port'
19
+ def initialize(message='Unknown host or port')
20
20
  super message, 'UNKNOWN_HOST'
21
21
  end
22
22
  end
23
23
 
24
24
  class IOError < PbError
25
- def initialize message='IO Error occurred'
25
+ def initialize(message='IO Error occurred')
26
26
  super message, 'IO_ERROR'
27
27
  end
28
28
  end
@@ -1,4 +1,3 @@
1
- # require 'protobuf'
2
1
  require 'protobuf/common/logger'
3
2
  require 'protobuf/rpc/rpc.pb'
4
3
  require 'protobuf/rpc/buffer'
@@ -7,52 +6,46 @@ require 'protobuf/rpc/stat'
7
6
 
8
7
  module Protobuf
9
8
  module Rpc
10
- module Server
11
-
9
+ module Server
10
+
12
11
  # Invoke the service method dictated by the proto wrapper request object
13
12
  def handle_client
14
- @stats.request_size = @buffer.size
15
-
16
- # Setup the initial request and response
17
- @request = Protobuf::Socketrpc::Request.new
18
- @response = Protobuf::Socketrpc::Response.new
19
-
20
13
  # Parse the protobuf request from the socket
21
- log_debug "[#{log_signature}] Parsing request from client"
14
+ log_debug { "[#{log_signature}] Parsing request from client" }
22
15
  parse_request_from_buffer
23
-
16
+
24
17
  # Determine the service class and method name from the request
25
- log_debug "[#{log_signature}] Extracting procedure call info from request"
18
+ log_debug { "[#{log_signature}] Extracting procedure call info from request" }
26
19
  parse_service_info
27
-
20
+
28
21
  # Call the service method
29
- log_debug "[#{log_signature}] Dispatching client request to service"
22
+ log_debug { "[#{log_signature}] Dispatching client request to service" }
30
23
  invoke_rpc_method
31
24
  rescue => error
32
25
  # Ensure we're handling any errors that try to slip out the back door
33
- log_error error.message
34
- log_error error.backtrace.join("\n")
26
+ log_error(error.message)
27
+ log_error(error.backtrace.join("\n"))
35
28
  handle_error(error)
36
29
  send_response
37
30
  end
38
31
 
39
32
  # Client error handler. Receives an exception object and writes it into the @response
40
33
  def handle_error(error)
41
- log_debug "[#{log_signature}] handle_error: %s" % error.inspect
34
+ log_debug { "[#{log_signature}] handle_error: %s" % error.inspect }
42
35
  if error.respond_to?(:to_response)
43
36
  error.to_response(@response)
44
37
  else
45
38
  message = error.respond_to?(:message) ? error.message : error.to_s
46
39
  code = error.respond_to?(:code) ? error.code.to_s : "RPC_ERROR"
47
- PbError.new(message, code).to_response(@response)
40
+ ::Protobuf::Rpc::PbError.new(message, code).to_response(@response)
48
41
  end
49
42
  end
50
-
43
+
51
44
  # Assuming all things check out, we can call the service method
52
45
  def invoke_rpc_method
53
46
  # Get a new instance of the service
54
47
  @service = @klass.new
55
-
48
+
56
49
  # Define our response callback to perform the "successful" response to our client
57
50
  # This decouples the service's rpc method from our response to the client,
58
51
  # allowing the service to be the dictator for when the response should be sent back.
@@ -65,30 +58,30 @@ module Protobuf
65
58
  send_response
66
59
  end
67
60
  end
68
-
61
+
69
62
  @service.on_rpc_failed do |error|
70
63
  unless @did_respond
71
64
  handle_error(error)
72
65
  send_response
73
66
  end
74
67
  end
75
-
68
+
76
69
  # Call the service method
77
- log_debug "[#{log_signature}] Invoking %s#%s with request %s" % [@klass.name, @method, @request.inspect]
70
+ log_debug { "[#{log_signature}] Invoking %s#%s with request %s" % [@klass.name, @method, @request.inspect] }
78
71
  @service.__send__(@method, @request)
79
72
  end
80
73
 
81
74
  def log_signature
82
75
  @log_signature ||= "server-#{self.class}"
83
76
  end
84
-
77
+
85
78
  # Parse the incoming request object into our expected request object
86
79
  def parse_request_from_buffer
87
- log_debug "[#{log_signature}] parsing request from buffer: %s" % @buffer.data.inspect
88
- @request.parse_from_string(@buffer.data)
80
+ log_debug { "[#{log_signature}] parsing request from buffer: %s" % @request_data }
81
+ @request.parse_from_string(@request_data)
89
82
  rescue => error
90
- exc = BadRequestData.new 'Unable to parse request: %s' % error.message
91
- log_error exc.message
83
+ exc = ::Protobuf::Rpc::BadRequestData.new 'Unable to parse request: %s' % error.message
84
+ log_error { exc.message }
92
85
  raise exc
93
86
  end
94
87
 
@@ -97,18 +90,18 @@ module Protobuf
97
90
  # response to the protobuf response wrapper
98
91
  def parse_response_from_service(response)
99
92
  expected = @klass.rpcs[@klass][@method].response_type
100
-
93
+
101
94
  # Cannibalize the response if it's a Hash
102
95
  response = expected.new(response) if response.is_a?(Hash)
103
96
  actual = response.class
104
- log_debug "[#{log_signature}] response (should/actual): %s/%s" % [expected.name, actual.name]
105
-
97
+ log_debug { "[#{log_signature}] response (should/actual): %s/%s" % [expected.name, actual.name] }
98
+
106
99
  # Determine if the service tried to change response types on us
107
100
  if expected == actual
108
101
  serialize_response(response)
109
102
  else
110
103
  # response types do not match, throw the appropriate error
111
- raise BadResponseProto, 'Response proto changed from %s to %s' % [expected.name, actual.name]
104
+ raise ::Protobuf::Rpc::BadResponseProto, 'Response proto changed from %s to %s' % [expected.name, actual.name]
112
105
  end
113
106
  rescue => error
114
107
  log_error error.message
@@ -118,13 +111,13 @@ module Protobuf
118
111
 
119
112
  # Parses and returns the service and method name from the request wrapper proto
120
113
  def parse_service_info
121
- @klass = Util.constantize(@request.service_name)
122
- @method = Util.underscore(@request.method_name).to_sym
114
+ @klass = ::Protobuf::Util.constantize(@request.service_name)
115
+ @method = ::Protobuf::Util.underscore(@request.method_name).to_sym
123
116
 
124
117
  unless @klass.instance_methods.include?(@method)
125
118
  raise MethodNotFound, "Service method #{@request.method_name} is not defined by the service"
126
119
  end
127
-
120
+
128
121
  @stats.service = @klass.name
129
122
  @stats.method = @method
130
123
  rescue NameError
@@ -134,24 +127,19 @@ module Protobuf
134
127
  # Write the response wrapper to the client
135
128
  def send_response
136
129
  raise 'Response already sent to client' if @did_respond
137
- log_debug "[#{log_signature}] Sending response to client: %s" % @response.inspect
138
- response_buffer = Protobuf::Rpc::Buffer.new(:write, @response)
139
- send_data(response_buffer.write)
140
- @stats.response_size = response_buffer.size
130
+ log_debug { "[#{log_signature}] Sending response to client: %s" % @response.inspect }
131
+ send_data
141
132
  @stats.end
142
133
  @stats.log_stats
143
134
  @did_respond = true
144
135
  end
145
136
 
146
137
  def serialize_response(response)
147
- log_debug "[#{log_signature}] serializing response: %s" % response.inspect
138
+ log_debug { "[#{log_signature}] serializing response: %s" % response.inspect }
148
139
  @response.response_proto = response.serialize_to_string
149
140
  rescue
150
141
  raise BadResponseProto, $!.message
151
142
  end
152
-
153
143
  end
154
-
155
144
  end
156
-
157
145
  end
@@ -0,0 +1,43 @@
1
+ require 'protobuf/rpc/server'
2
+
3
+ module Protobuf
4
+ module Rpc
5
+ module Evented
6
+ class Server < ::EventMachine::Connection
7
+ include ::Protobuf::Rpc::Server
8
+ include ::Protobuf::Logger::LogMethods
9
+
10
+ # Initialize a new read buffer for storing client request info
11
+ def post_init
12
+ log_debug { '[server] Post init, new read buffer created' }
13
+ @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
14
+ @stats.client = ::Socket.unpack_sockaddr_in(get_peername)
15
+ @request_buffer = Protobuf::Rpc::Buffer.new(:read)
16
+ @request = ::Protobuf::Socketrpc::Request.new
17
+ @response = ::Protobuf::Socketrpc::Response.new
18
+
19
+ @did_respond = false
20
+ end
21
+
22
+ # Receive a chunk of data, potentially flushed to handle_client
23
+ def receive_data(data)
24
+ log_debug { '[server] receive_data: %s' % data }
25
+
26
+ @request_buffer << data
27
+ @request_data = @request_buffer.data
28
+ @stats.request_size = @request_buffer.size
29
+
30
+ handle_client if @request_buffer.flushed?
31
+ end
32
+
33
+ def send_data
34
+ response_buffer = Protobuf::Rpc::Buffer.new(:write)
35
+ response_buffer.set_data(@response)
36
+ @stats.response_size = response_buffer.size
37
+ log_debug { "[#{log_signature}] sending data: #{response_buffer.inspect}" }
38
+ super(response_buffer.write)
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -20,7 +20,7 @@ module Protobuf
20
20
 
21
21
  # Startup and run the rpc server
22
22
  EM.schedule do
23
- EventMachine.start_server(server.host, server.port, Protobuf::Rpc::EventedServer) && \
23
+ EventMachine.start_server(server.host, server.port, Protobuf::Rpc::Evented::Server) && \
24
24
  Protobuf::Logger.info('RPC Server listening at %s:%d in %s' % [server.host, server.port, server.env])
25
25
  end
26
26
 
@@ -0,0 +1,108 @@
1
+ require 'protobuf/rpc/server'
2
+ require 'protobuf/rpc/servers/socket/worker'
3
+ module Protobuf
4
+ module Rpc
5
+ module Socket
6
+
7
+ class Server
8
+ include ::Protobuf::Rpc::Server
9
+ include ::Protobuf::Logger::LogMethods
10
+
11
+ def self.cleanup?
12
+ # every 10 connections run a cleanup routine after closing the response
13
+ @threads.size > (@thread_threshold - 1) && (@threads.size % @thread_threshold) == 0
14
+ end
15
+
16
+ def self.cleanup_threads
17
+ log_debug { "[#{log_signature}] Thread cleanup - #{@threads.size} - start" }
18
+
19
+ @threads = @threads.select do |t|
20
+ if t[:thread].alive?
21
+ true
22
+ else
23
+ t[:thread].join
24
+ @working.delete(t[:socket])
25
+ false
26
+ end
27
+ end
28
+
29
+ log_debug { "[#{log_signature}] Thread cleanup - #{@threads.size} - complete" }
30
+ end
31
+
32
+ def self.log_signature
33
+ @log_signature ||= "server-#{self}"
34
+ end
35
+
36
+ def self.new_worker(socket)
37
+ Thread.new(socket) do |sock|
38
+ ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
39
+ s.close
40
+ end
41
+ end
42
+ end
43
+
44
+ def self.run(opts = {})
45
+ log_debug { "[#{log_signature}] Run" }
46
+ host = opts.fetch(:host, "127.0.0.1")
47
+ port = opts.fetch(:port, 9399)
48
+ backlog = opts.fetch(:backlog, 100)
49
+ thread_threshold = opts.fetch(:thread_threshold, 100)
50
+ auto_collect_timeout = opts.fetch(:auto_collect_timeout, 20)
51
+
52
+ @threads = []
53
+ @thread_threshold = thread_threshold
54
+ @server = ::TCPServer.new(host, port)
55
+ @server.listen(backlog)
56
+ @working = []
57
+ @listen_fds = [@server]
58
+ @running = true
59
+
60
+ while running?
61
+ log_debug { "[#{log_signature}] Waiting for connections" }
62
+
63
+ if ready_cnxns = IO.select(@listen_fds, [], [], auto_collect_timeout)
64
+ cnxns = ready_cnxns.first
65
+ cnxns.each do |client|
66
+ case
67
+ when !running? then
68
+ # no-op
69
+ when client == @server then
70
+ log_debug { "[#{log_signature}] Accepted new connection" }
71
+ client, sockaddr = @server.accept
72
+ @listen_fds << client
73
+ else
74
+ if !@working.include?(client)
75
+ @working << @listen_fds.delete(client)
76
+ log_debug { "[#{log_signature}] Working" }
77
+ @threads << { :thread => new_worker(client), :socket => client }
78
+
79
+ cleanup_threads if cleanup?
80
+ end
81
+ end
82
+ end
83
+ else
84
+ # Run a cleanup if select times out while waiting
85
+ cleanup_threads if @threads.size > 1
86
+ end
87
+ end
88
+
89
+ rescue Errno::EADDRINUSE
90
+ raise
91
+ rescue
92
+ # Closing the server causes the loop to raise an exception here
93
+ raise if running?
94
+ end
95
+
96
+ def self.running?
97
+ @running
98
+ end
99
+
100
+ def self.stop
101
+ @running = false
102
+ @server.close if @server
103
+ end
104
+ end
105
+
106
+ end
107
+ end
108
+ end