protobuf 1.1.3 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. data/.gitignore +3 -0
  2. data/Gemfile.lock +44 -25
  3. data/README.md +2 -2
  4. data/Rakefile +15 -0
  5. data/bin/rpc_server +52 -11
  6. data/lib/protobuf.rb +22 -10
  7. data/lib/protobuf/common/logger.rb +26 -25
  8. data/lib/protobuf/descriptor/file_descriptor.rb +1 -1
  9. data/lib/protobuf/message/field.rb +2 -2
  10. data/lib/protobuf/rpc/buffer.rb +30 -25
  11. data/lib/protobuf/rpc/client.rb +8 -8
  12. data/lib/protobuf/rpc/connector.rb +2 -0
  13. data/lib/protobuf/rpc/connectors/base.rb +0 -1
  14. data/lib/protobuf/rpc/connectors/common.rb +48 -48
  15. data/lib/protobuf/rpc/connectors/em_client.rb +53 -27
  16. data/lib/protobuf/rpc/connectors/eventmachine.rb +14 -17
  17. data/lib/protobuf/rpc/connectors/socket.rb +23 -16
  18. data/lib/protobuf/rpc/connectors/zmq.rb +73 -0
  19. data/lib/protobuf/rpc/error.rb +1 -2
  20. data/lib/protobuf/rpc/error/client_error.rb +4 -4
  21. data/lib/protobuf/rpc/server.rb +31 -43
  22. data/lib/protobuf/rpc/servers/evented/server.rb +43 -0
  23. data/lib/protobuf/rpc/servers/evented_runner.rb +1 -1
  24. data/lib/protobuf/rpc/servers/socket/server.rb +108 -0
  25. data/lib/protobuf/rpc/servers/socket/worker.rb +59 -0
  26. data/lib/protobuf/rpc/servers/socket_runner.rb +3 -3
  27. data/lib/protobuf/rpc/servers/zmq/broker.rb +85 -0
  28. data/lib/protobuf/rpc/servers/zmq/server.rb +50 -0
  29. data/lib/protobuf/rpc/servers/zmq/util.rb +27 -0
  30. data/lib/protobuf/rpc/servers/zmq/worker.rb +72 -0
  31. data/lib/protobuf/rpc/servers/zmq_runner.rb +26 -0
  32. data/lib/protobuf/rpc/service.rb +5 -5
  33. data/lib/protobuf/version.rb +1 -1
  34. data/protobuf.gemspec +12 -10
  35. data/spec/benchmark/tasks.rb +37 -5
  36. data/spec/functional/evented_server_spec.rb +64 -0
  37. data/spec/functional/socket_server_spec.rb +63 -0
  38. data/spec/functional/zmq_server_spec.rb +63 -0
  39. data/spec/helper/server.rb +32 -12
  40. data/spec/lib/protobuf/message/encoder_spec.rb +19 -0
  41. data/spec/proto/test.pb.rb +3 -3
  42. data/spec/proto/test.proto +3 -3
  43. data/spec/proto/test_service.rb +1 -0
  44. data/spec/spec_helper.rb +6 -0
  45. data/spec/unit/message_spec.rb +1 -1
  46. data/spec/unit/rpc/client_spec.rb +11 -3
  47. data/spec/unit/rpc/connectors/common_spec.rb +0 -1
  48. data/spec/unit/rpc/connectors/eventmachine_client_spec.rb +32 -0
  49. data/spec/unit/rpc/connectors/socket_spec.rb +2 -4
  50. data/spec/unit/rpc/connectors/zmq_spec.rb +27 -0
  51. data/spec/unit/rpc/servers/evented_server_spec.rb +3 -3
  52. data/spec/unit/rpc/servers/socket_server_spec.rb +14 -13
  53. data/spec/unit/rpc/servers/zmq/broker_spec.rb +27 -0
  54. data/spec/unit/rpc/servers/zmq/server_spec.rb +37 -0
  55. data/spec/unit/rpc/servers/zmq/util_spec.rb +41 -0
  56. data/spec/unit/rpc/servers/zmq/worker_spec.rb +36 -0
  57. data/spec/unit/rpc/service_spec.rb +22 -18
  58. metadata +87 -40
  59. data/lib/protobuf/rpc/servers/evented_server.rb +0 -28
  60. data/lib/protobuf/rpc/servers/socket_server.rb +0 -146
@@ -0,0 +1,59 @@
1
+ module Protobuf
2
+ module Rpc
3
+ module Socket
4
+
5
+ class Worker
6
+ include ::Protobuf::Rpc::Server
7
+ include ::Protobuf::Logger::LogMethods
8
+
9
+ def initialize(sock, &complete_cb)
10
+ @did_response = false
11
+ @socket = sock
12
+ @request = Protobuf::Socketrpc::Request.new
13
+ @response = Protobuf::Socketrpc::Response.new
14
+ request_buffer = Protobuf::Rpc::Buffer.new(:read)
15
+ @stats = Protobuf::Rpc::Stat.new(:SERVER, true)
16
+ @complete_cb = complete_cb
17
+ log_debug { "[#{log_signature}] Post init, new read buffer created" }
18
+
19
+ @stats.client = ::Socket.unpack_sockaddr_in(@socket.getpeername)
20
+ log_debug { "stats are #{@stats.to_s}" }
21
+ request_buffer << read_data
22
+ @request_data = request_buffer.data
23
+
24
+ @stats.request_size = request_buffer.size
25
+
26
+ log_debug { "[#{log_signature}] handling request" }
27
+ handle_client if request_buffer.flushed?
28
+ end
29
+
30
+ def log_signature
31
+ @log_signature ||= "server-#{self.class}-#{object_id}"
32
+ end
33
+
34
+ def read_data
35
+ size_io = StringIO.new
36
+
37
+ while((size_reader = @socket.getc) != "-")
38
+ size_io << size_reader
39
+ end
40
+ str_size_io = size_io.string
41
+
42
+ "#{str_size_io}-#{@socket.read(str_size_io.to_i)}"
43
+ end
44
+
45
+ def send_data
46
+ raise 'Socket closed unexpectedly' if(@socket.nil? || @socket.closed?)
47
+ response_buffer = Protobuf::Rpc::Buffer.new(:write)
48
+ response_buffer.set_data(@response)
49
+ @stats.response_size = response_buffer.size
50
+ log_debug { "[#{log_signature}] sending data : %s" % response_buffer.data }
51
+ @socket.write(response_buffer.write)
52
+ @socket.flush
53
+ @complete_cb.call(@socket)
54
+ end
55
+ end
56
+
57
+ end
58
+ end
59
+ end
@@ -3,12 +3,12 @@ module Protobuf
3
3
  class SocketRunner
4
4
 
5
5
  def self.stop
6
- Protobuf::Rpc::SocketServer.stop
6
+ Protobuf::Rpc::Socket::Server.stop
7
7
  Protobuf::Logger.info 'Shutdown complete'
8
8
  end
9
9
 
10
10
  def self.run(server)
11
- Protobuf::Logger.info "SocketServer Running"
11
+ Protobuf::Logger.info "Socket::Server Running"
12
12
  server_config = case
13
13
  when server.is_a?(OpenStruct) then
14
14
  server.marshal_dump
@@ -20,7 +20,7 @@ module Protobuf
20
20
  raise "Cannot parser Socket Server - server options"
21
21
  end
22
22
 
23
- Protobuf::Rpc::SocketServer.run(server_config)
23
+ Protobuf::Rpc::Socket::Server.run(server_config)
24
24
  end
25
25
  end
26
26
 
@@ -0,0 +1,85 @@
1
+ require 'resolv'
2
+ require 'protobuf/rpc/servers/zmq/util'
3
+
4
+ module Protobuf
5
+ module Rpc
6
+ module Zmq
7
+ class Broker
8
+ include ::Protobuf::Rpc::Zmq::Util
9
+ attr_reader :frontend, :backend, :poller, :context
10
+
11
+ ##
12
+ # Constructor
13
+ #
14
+ def initialize(opts={})
15
+ @context = ::ZMQ::Context.new
16
+ @frontend = setup_frontend(opts)
17
+ @backend = setup_backend(opts)
18
+ @poller = setup_poller
19
+ end
20
+
21
+ ##
22
+ # Instance Methods
23
+ #
24
+ def poll
25
+ poller.poll(1000)
26
+ poller.readables.each do |socket|
27
+ case socket
28
+ when frontend then
29
+ move_to(backend, socket)
30
+ when backend then
31
+ move_to(frontend, socket)
32
+ end
33
+ end
34
+ end
35
+
36
+ def teardown
37
+ frontend.close
38
+ backend.close
39
+ context.terminate
40
+ end
41
+
42
+ private
43
+
44
+ def move_to(frontend_or_backend, socket)
45
+ more_data = true
46
+
47
+ while more_data do
48
+ socket.recv_string(data = "")
49
+ more_data = socket.more_parts?
50
+ more_data_flag = (more_data ? ::ZMQ::SNDMORE : 0)
51
+ frontend_or_backend.send_string(data, more_data_flag)
52
+ end
53
+ end
54
+
55
+ def setup_backend(opts={})
56
+ dealer_options = opts.merge(:port => opts.fetch(:port, 9399) + 1)
57
+ host = dealer_options.fetch(:host, "127.0.0.1")
58
+ port = dealer_options.fetch(:port, 9400)
59
+ protocol = dealer_options.fetch(:protocol, "tcp")
60
+
61
+ zmq_backend = context.socket(::ZMQ::DEALER)
62
+ zmq_error_check(zmq_backend.bind("#{protocol}://#{resolve_ip(host)}:#{port}"))
63
+ zmq_backend
64
+ end
65
+
66
+ def setup_frontend(opts={})
67
+ host = opts.fetch(:host, "127.0.0.1")
68
+ port = opts.fetch(:port, 9399)
69
+ protocol = opts.fetch(:protocol, "tcp")
70
+
71
+ zmq_frontend = context.socket(::ZMQ::ROUTER)
72
+ zmq_error_check(zmq_frontend.bind("#{protocol}://#{resolve_ip(host)}:#{port}"))
73
+ zmq_frontend
74
+ end
75
+
76
+ def setup_poller
77
+ zmq_poller = ::ZMQ::Poller.new
78
+ zmq_poller.register(frontend, ::ZMQ::POLLIN)
79
+ zmq_poller.register(backend, ::ZMQ::POLLIN)
80
+ zmq_poller
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,50 @@
1
+ require 'protobuf/rpc/servers/zmq/broker'
2
+ require 'protobuf/rpc/servers/zmq/worker'
3
+ require 'protobuf/rpc/servers/zmq/util'
4
+
5
+ module Protobuf
6
+ module Rpc
7
+ module Zmq
8
+ class Server
9
+ include ::Protobuf::Rpc::Zmq::Util
10
+
11
+ ##
12
+ # Class Methods
13
+ #
14
+ def self.run(opts = {})
15
+ log_debug { "[#{log_signature}] initializing broker" }
16
+ @broker = ::Protobuf::Rpc::Zmq::Broker.new(opts)
17
+ local_worker_threads = opts.fetch(:threads, 5)
18
+
19
+ worker_options = opts.merge(:port => opts.fetch(:port, 9399) + 1)
20
+ log_debug { "[#{log_signature}] starting server workers" }
21
+ local_worker_threads.times do
22
+ @threads << Thread.new { ::Protobuf::Rpc::Zmq::Worker.new(worker_options).run }
23
+ end
24
+ @running = true
25
+
26
+ log_debug { "[#{log_signature}] server started" }
27
+ while self.running? do
28
+ @broker.poll
29
+ end
30
+ ensure
31
+ @broker.teardown if @broker
32
+ end
33
+
34
+ def self.running?
35
+ !!@running
36
+ end
37
+
38
+ def self.stop
39
+ @running = false
40
+
41
+ @threads.each do |t|
42
+ t.join
43
+ end
44
+ end
45
+
46
+ @threads ||= []
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,27 @@
1
+ module Protobuf
2
+ module Rpc
3
+ module Zmq
4
+
5
+ module Util
6
+ include ::Protobuf::Logger::LogMethods
7
+ def self.included(base)
8
+ base.extend(::Protobuf::Rpc::Zmq::Util)
9
+ end
10
+
11
+ def zmq_error_check(return_code)
12
+ raise "Last API call failed with \"#{::ZMQ::Util.error_string}\"#{$/}#{$/}#{caller(1)}" unless return_code >= 0
13
+ end
14
+
15
+ def log_signature
16
+ @log_signature ||= "server-#{self.class}-#{object_id}"
17
+ end
18
+
19
+ def resolve_ip(hostname)
20
+ ::Resolv.getaddress(hostname)
21
+ end
22
+
23
+ end
24
+
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,72 @@
1
+ require 'protobuf/rpc/server'
2
+ require 'protobuf/rpc/servers/zmq/util'
3
+ module Protobuf
4
+ module Rpc
5
+ module Zmq
6
+
7
+ class Worker
8
+ include ::Protobuf::Rpc::Server
9
+ include ::Protobuf::Rpc::Zmq::Util
10
+
11
+ ##
12
+ # Constructor
13
+ #
14
+ def initialize(opts={})
15
+
16
+ @options = opts
17
+ host = @options.fetch(:host, "127.0.0.1")
18
+ port = @options.fetch(:port, 9400)
19
+ protocol = @options.fetch(:protocol, "tcp")
20
+
21
+ @zmq_context = ::ZMQ::Context.new
22
+ @socket = @zmq_context.socket(::ZMQ::REP)
23
+ zmq_error_check(@socket.connect("#{protocol}://#{resolve_ip(host)}:#{port}"))
24
+
25
+ @poller = ::ZMQ::Poller.new
26
+ @poller.register(@socket, ::ZMQ::POLLIN)
27
+ end
28
+
29
+ ##
30
+ # Instance Methods
31
+ #
32
+ def handle_request(socket)
33
+ @request_data = ''
34
+ zmq_error_check(socket.recv_string(@request_data))
35
+ log_debug { "[#{log_signature}] handling request" } if(!@request_data.nil?)
36
+ end
37
+
38
+ def initialize_buffers
39
+ @did_respond = false
40
+ @request = ::Protobuf::Socketrpc::Request.new
41
+ @response = ::Protobuf::Socketrpc::Response.new
42
+ @stats = ::Protobuf::Rpc::Stat.new(:SERVER, true)
43
+ log_debug { "[#{log_signature}] Post init" }
44
+ end
45
+
46
+ def run
47
+ while ::Protobuf::Rpc::Zmq::Server.running? do
48
+ # poll for 1_000 milliseconds then continue looping
49
+ # This lets us see whether we need to die
50
+ @poller.poll(1_000)
51
+ @poller.readables.each do |socket|
52
+ initialize_buffers
53
+ handle_request(socket)
54
+ handle_client unless(@request_data.nil?)
55
+ end
56
+ end
57
+ ensure
58
+ @socket.close
59
+ @zmq_context.terminate
60
+ end
61
+
62
+ def send_data
63
+ response_data = @response.is_a?(Protobuf::Message) ? @response.serialize_to_string : @response.to_s
64
+ @stats.response_size = response_data.size
65
+ zmq_error_check(@socket.send_string(response_data))
66
+ @did_respond = true
67
+ end
68
+ end
69
+
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,26 @@
1
+ module Protobuf
2
+ module Rpc
3
+ class ZmqRunner
4
+
5
+ def self.stop
6
+ Protobuf::Rpc::Zmq::Server.stop
7
+ Protobuf::Logger.info('Shutdown complete')
8
+ end
9
+
10
+ def self.run(server)
11
+ Protobuf::Logger.info "ZmqServer Running"
12
+ server_config = case
13
+ when server.is_a?(OpenStruct) then
14
+ server.marshal_dump
15
+ when server.respond_to?(:to_hash) then
16
+ server.to_hash
17
+ else
18
+ raise "Cannot parser Zmq Server - server options"
19
+ end
20
+
21
+ Protobuf::Rpc::Zmq::Server.run(server_config)
22
+ end
23
+ end
24
+
25
+ end
26
+ end
@@ -130,7 +130,7 @@ module Protobuf
130
130
  log_error exc.message
131
131
  raise exc
132
132
  else
133
- log_error "-------------- [#{log_signature}] %s#%s not rpc method, passing to super" % [self.class.name, m.to_s]
133
+ log_error { "-------------- [#{log_signature}] %s#%s not rpc method, passing to super" % [self.class.name, m.to_s] }
134
134
  super m, params
135
135
  end
136
136
  end
@@ -221,17 +221,17 @@ module Protobuf
221
221
  # Setup the response
222
222
  @response = rpcs[method].response_type.new
223
223
 
224
- log_debug "[#{log_signature}] calling service method %s#%s" % [self.class, method]
224
+ log_debug { "[#{log_signature}] calling service method %s#%s" % [self.class, method] }
225
225
  # Call the aliased rpc method (e.g. :rpc_find for :find)
226
226
  __send__("rpc_#{method}".to_sym)
227
- log_debug "[#{log_signature}] completed service method %s#%s" % [self.class, method]
227
+ log_debug { "[#{log_signature}] completed service method %s#%s" % [self.class, method] }
228
228
 
229
229
  # Pass the populated response back to the server
230
230
  # Note this will only get called if the rpc method didn't explode (by design)
231
231
  if @async_responder
232
- log_debug "[#{log_signature}] async request, not sending response (yet)"
232
+ log_debug { "[#{log_signature}] async request, not sending response (yet)" }
233
233
  else
234
- log_debug "[#{log_signature}] trigger server send_response"
234
+ log_debug { "[#{log_signature}] trigger server send_response" }
235
235
  send_response
236
236
  end
237
237
  end
@@ -1,3 +1,3 @@
1
1
  module Protobuf
2
- VERSION = '1.1.3'
2
+ VERSION = '1.3.0'
3
3
  end
@@ -12,19 +12,21 @@ Gem::Specification.new do |s|
12
12
  s.homepage = %q{https://github.com/localshred/protobuf}
13
13
  s.summary = 'Ruby implementation for Protocol Buffers. Works with other protobuf rpc implementations (e.g. Java, Python, C++).'
14
14
  s.description = s.summary + "\n\nThis gem has diverged from https://github.com/macks/ruby-protobuf. All credit for serialization and rprotoc work most certainly goes to the original authors. All RPC implementation code (client/server/service) was written and is maintained by this author. Attempts to reconcile the original codebase with the current RPC implementation went unsuccessful."
15
-
15
+
16
16
  s.files = `git ls-files`.split("\n")
17
17
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
18
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
19
  s.require_paths = ["lib"]
20
-
21
- s.add_dependency 'eventmachine', '~> 0.12.10'
22
- s.add_dependency 'eventually', '~> 0.1.0'
23
- s.add_dependency 'json_pure', '~> 1.6.4'
24
20
 
25
- s.add_development_dependency 'rake', '~> 0.8.7'
26
- s.add_development_dependency 'rspec', '~> 2.8.0'
27
- s.add_development_dependency 'yard', '~> 0.7.4'
28
- s.add_development_dependency 'redcarpet', '~> 1.17.2'
29
- s.add_development_dependency 'simplecov', '~> 0.5.4'
21
+ s.add_dependency 'eventmachine'
22
+ s.add_dependency 'eventually'
23
+ s.add_dependency 'json_pure'
24
+ s.add_dependency 'ffi-rzmq'
25
+
26
+ s.add_development_dependency 'rake'
27
+ s.add_development_dependency 'pry'
28
+ s.add_development_dependency 'pry-nav'
29
+ s.add_development_dependency 'rspec'
30
+ s.add_development_dependency 'yard'
31
+ s.add_development_dependency 'simplecov'
30
32
  end
@@ -2,6 +2,13 @@ require 'benchmark'
2
2
  require 'helper/all'
3
3
  require 'proto/test_service_impl'
4
4
 
5
+ # Including a way to turn on debug logger for spec runs
6
+ if ENV["DEBUG"]
7
+ puts 'debugging'
8
+ debug_log = File.expand_path('../debug_bench.log', File.dirname(__FILE__) )
9
+ Protobuf::Logger.configure(:file => debug_log, :level => ::Logger::DEBUG)
10
+ end
11
+
5
12
  namespace :benchmark do
6
13
  include SilentConstants
7
14
 
@@ -37,7 +44,7 @@ namespace :benchmark do
37
44
  EM.stop if EM.reactor_running?
38
45
 
39
46
  EventMachine.fiber_run do
40
- StubServer.new(:server => Protobuf::Rpc::SocketServer, :port => 9399) do |server|
47
+ StubServer.new(:server => Protobuf::Rpc::Socket::Server, :port => 9399) do |server|
41
48
  client = Spec::Proto::TestService.client(:async => false, :port => 9399)
42
49
 
43
50
  benchmark_wrapper(global_bench) do |bench|
@@ -54,7 +61,7 @@ namespace :benchmark do
54
61
  def sock_client_sock_server(number_tests, test_length, global_bench = nil)
55
62
  EM.stop if EM.reactor_running?
56
63
 
57
- StubServer.new(:server => Protobuf::Rpc::SocketServer, :port => 9399) do |server|
64
+ StubServer.new(:server => Protobuf::Rpc::Socket::Server, :port => 9399) do |server|
58
65
  with_constants "Protobuf::ClientType" => "Socket" do
59
66
  client = Spec::Proto::TestService.client(:async => false, :port => 9399)
60
67
 
@@ -88,6 +95,30 @@ namespace :benchmark do
88
95
  Thread.kill(em_thread) if em_thread
89
96
  end
90
97
 
98
+ def zmq_client_zmq_server(number_tests, test_length, global_bench = nil)
99
+ require 'ffi-rzmq'
100
+ require 'protobuf/rpc/connectors/zmq'
101
+ require 'protobuf/rpc/servers/zmq/server'
102
+ StubServer.new(:port => 9399, :server => Protobuf::Rpc::Zmq::Server) do |server|
103
+ with_constants "Protobuf::ClientType" => "Zmq" do
104
+ client = Spec::Proto::TestService.client(:async => false, :port => 9399)
105
+
106
+ benchmark_wrapper(global_bench) do |bench|
107
+ bench.report("ZS / ZC") do
108
+ (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
109
+ end
110
+ end
111
+ end
112
+ server.stop
113
+ end
114
+ end
115
+
116
+ desc "benchmark ZMQ client with ZMQ server"
117
+ task :zmq_client_zmq_server, [:number, :length] do |t, args|
118
+ args.with_defaults(:number => 1000, :length => 100)
119
+ zmq_client_zmq_server(args[:number], args[:length])
120
+ end
121
+
91
122
  desc "benchmark EventMachine client with EventMachine server"
92
123
  task :em_client_em_server, [:number, :length] do |t, args|
93
124
  args.with_defaults(:number => 1000, :length => 100)
@@ -117,10 +148,11 @@ namespace :benchmark do
117
148
  args.with_defaults(:number => 1000, :length => 100)
118
149
 
119
150
  Benchmark.bm(10) do |bench|
120
- em_client_em_server(args[:number], args[:length], bench)
121
- em_client_sock_server(args[:number], args[:length], bench)
151
+ # zmq_client_zmq_server(args[:number], args[:length], bench)
152
+ # em_client_em_server(args[:number], args[:length], bench)
153
+ # em_client_sock_server(args[:number], args[:length], bench)
122
154
  sock_client_sock_server(args[:number], args[:length], bench)
123
- sock_client_em_server(args[:number], args[:length], bench)
155
+ # sock_client_em_server(args[:number], args[:length], bench)
124
156
  end
125
157
  end
126
158
  end