protobuf 2.7.12 → 2.8.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. data/README.md +39 -2
  2. data/lib/protobuf.rb +17 -26
  3. data/lib/protobuf/cli.rb +106 -86
  4. data/lib/protobuf/field/bytes_field.rb +6 -8
  5. data/lib/protobuf/field/float_field.rb +5 -1
  6. data/lib/protobuf/field/string_field.rb +7 -8
  7. data/lib/protobuf/rpc/connectors/base.rb +1 -1
  8. data/lib/protobuf/rpc/connectors/zmq.rb +157 -29
  9. data/lib/protobuf/rpc/dynamic_discovery.pb.rb +49 -0
  10. data/lib/protobuf/rpc/error/client_error.rb +5 -5
  11. data/lib/protobuf/rpc/error/server_error.rb +7 -7
  12. data/lib/protobuf/rpc/rpc.pb.rb +13 -12
  13. data/lib/protobuf/rpc/servers/evented_runner.rb +11 -6
  14. data/lib/protobuf/rpc/servers/socket/server.rb +19 -15
  15. data/lib/protobuf/rpc/servers/socket_runner.rb +21 -18
  16. data/lib/protobuf/rpc/servers/zmq/broker.rb +104 -94
  17. data/lib/protobuf/rpc/servers/zmq/server.rb +263 -43
  18. data/lib/protobuf/rpc/servers/zmq/util.rb +18 -6
  19. data/lib/protobuf/rpc/servers/zmq/worker.rb +102 -39
  20. data/lib/protobuf/rpc/servers/zmq_runner.rb +31 -20
  21. data/lib/protobuf/rpc/service.rb +24 -12
  22. data/lib/protobuf/rpc/service_directory.rb +206 -0
  23. data/lib/protobuf/rpc/stat.rb +1 -1
  24. data/lib/protobuf/version.rb +1 -1
  25. data/proto/dynamic_discovery.proto +44 -0
  26. data/spec/benchmark/tasks.rb +1 -3
  27. data/spec/functional/socket_server_spec.rb +6 -5
  28. data/spec/functional/zmq_server_spec.rb +59 -30
  29. data/spec/lib/protobuf/cli_spec.rb +49 -54
  30. data/spec/lib/protobuf/enum_spec.rb +1 -1
  31. data/spec/lib/protobuf/rpc/client_spec.rb +1 -1
  32. data/spec/lib/protobuf/rpc/connectors/zmq_spec.rb +43 -1
  33. data/spec/lib/protobuf/rpc/servers/evented_server_spec.rb +2 -1
  34. data/spec/lib/protobuf/rpc/servers/socket_server_spec.rb +9 -8
  35. data/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb +24 -19
  36. data/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb +5 -5
  37. data/spec/lib/protobuf/rpc/service_directory_spec.rb +183 -0
  38. data/spec/support/server.rb +21 -12
  39. data/spec/support/test/resource.pb.rb +6 -0
  40. data/spec/support/test/resource.proto +5 -0
  41. data/spec/support/test/resource_service.rb +7 -0
  42. metadata +70 -38
  43. checksums.yaml +0 -7
  44. data/spec/lib/protobuf/field/string_field_spec.rb +0 -46
  45. data/spec/lib/protobuf/rpc/servers/zmq/broker_spec.rb +0 -31
data/README.md CHANGED
@@ -303,8 +303,7 @@ any other filter calls which would run afterwards, as well as canceling
303
303
  invocation of the service method. Note: You must actually return false,
304
304
  not just a "falsey" value such as nil.
305
305
 
306
- __After Filters__ – There is no request shortcutting since the after
307
- filter runs after the request. Duh.
306
+ __After Filters__ – No request shortcutting.
308
307
 
309
308
  #### Filter options
310
309
 
@@ -415,6 +414,44 @@ Many different options can be passed to the `.client` call above
415
414
  (such as `:timeout => 600`). See the `lib/protobuf/rpc/client.rb`
416
415
  and `lib/protobuf/rpc/service.rb` files for more documentation.
417
416
 
417
+ ### Dynamic Discovery (ZMQ Only)
418
+ It is possible to setup the RPC server and client in a way that
419
+ allows servers to be dynamically discovered by the client.
420
+
421
+ #### In the client
422
+ ```ruby
423
+ ServiceDirectory.start do |config|
424
+ config.port = 53000
425
+ end
426
+
427
+ # If your server also runs this code, it will default to the
428
+ # given port when sending beacons and have its own service
429
+ # directory. You can prevent this code from running on the
430
+ # server if needed:
431
+ unless defined? ::Protobuf::CLI
432
+ ServiceDirectory.start do |config|
433
+ config.port = 53000
434
+ end
435
+ end
436
+ ```
437
+
438
+ #### Starting the server
439
+ ```
440
+ $ rpc_server --broadcast-beacons --beacon-port 53000 ...
441
+ ```
442
+
443
+ The client will listen on the specified port for beacons broadcast
444
+ by servers. Each beacon includes a list of services provided by the
445
+ broadcasting server. The client randomly selects a server for the
446
+ desired service each time a request is made.
447
+
448
+ __CAUTION:__ When running multiple environments on a single network,
449
+ e.g., qa and staging, be sure that each environment is setup with
450
+ a unique beacon port; otherwise, clients in one environment _will_
451
+ make requests to servers in the other environment.
452
+
453
+ Check out {Protobuf::ServiceDirectory} for more details.
454
+
418
455
  ## 3. RPC Interop
419
456
 
420
457
  The main reason I wrote this gem was to provide a ruby implementation
@@ -3,16 +3,7 @@ require 'socket'
3
3
  require 'pp'
4
4
  require 'stringio'
5
5
  require 'active_support/core_ext/object/blank'
6
- require 'active_support/version'
7
-
8
- if ActiveSupport::VERSION::MAJOR > 2
9
- require 'active_support/core_ext/object/try'
10
- else
11
- require 'active_support/core_ext/module/delegation'
12
- require 'active_support/core_ext/kernel/reporting'
13
- require 'active_support/core_ext/try'
14
- end
15
-
6
+ require 'active_support/core_ext/object/try'
16
7
  require 'active_support/inflector'
17
8
  require 'active_support/json'
18
9
 
@@ -24,7 +15,7 @@ module Protobuf
24
15
  # Default is Socket as it has no external dependencies.
25
16
  DEFAULT_CONNECTOR = :socket
26
17
 
27
- module_function
18
+ module_function
28
19
 
29
20
  # Client Host
30
21
  #
@@ -62,14 +53,14 @@ module Protobuf
62
53
  # the Garbage Collector when handling an rpc request.
63
54
  # Once the request is completed, the GC is enabled again.
64
55
  # This optomization provides a huge boost in speed to rpc requests.
65
- def self.gc_pause_server_request?
66
- return @_gc_pause_server_request unless @_gc_pause_server_request.nil?
67
- gc_pause_server_request = false
68
- end
56
+ def self.gc_pause_server_request?
57
+ return @_gc_pause_server_request unless @_gc_pause_server_request.nil?
58
+ gc_pause_server_request = false
59
+ end
69
60
 
70
- def self.gc_pause_server_request=(value)
71
- @_gc_pause_server_request = !!value
72
- end
61
+ def self.gc_pause_server_request=(value)
62
+ @_gc_pause_server_request = !!value
63
+ end
73
64
 
74
65
  # Print Deprecation Warnings
75
66
  #
@@ -81,14 +72,14 @@ module Protobuf
81
72
  # ENV['PB_IGNORE_DEPRECATIONS'] to a non-empty value.
82
73
  #
83
74
  # The rpc_server option will override the ENV setting.
84
- def self.print_deprecation_warnings?
85
- return @_print_deprecation_warnings unless @_print_deprecation_warnings.nil?
86
- print_deprecation_warnings = ENV.key?('PB_IGNORE_DEPRECATIONS') ? false : true
87
- end
88
-
89
- def self.print_deprecation_warnings=(value)
90
- @_print_deprecation_warnings = !!value
91
- end
75
+ def self.print_deprecation_warnings?
76
+ return @_print_deprecation_warnings unless @_print_deprecation_warnings.nil?
77
+ print_deprecation_warnings = ENV.key?('PB_IGNORE_DEPRECATIONS') ? false : true
78
+ end
79
+
80
+ def self.print_deprecation_warnings=(value)
81
+ @_print_deprecation_warnings = !!value
82
+ end
92
83
 
93
84
  end
94
85
 
@@ -9,7 +9,7 @@ module Protobuf
9
9
  class CLI < ::Thor
10
10
  include ::Thor::Actions
11
11
 
12
- attr_accessor :runner, :mode, :start_aborted
12
+ attr_accessor :runner, :mode
13
13
 
14
14
  default_task :start
15
15
 
@@ -29,6 +29,10 @@ module Protobuf
29
29
  option :evented, :type => :boolean, :aliases => %w(-m), :desc => 'Evented Mode for server and client connections (uses EventMachine).'
30
30
  option :zmq, :type => :boolean, :aliases => %w(-z), :desc => 'ZeroMQ Socket Mode for server and client connections.'
31
31
 
32
+ option :beacon_address, :type => :string, :desc => 'Broadcast beacons to this address (defaul: value of ServiceDirectory.address)'
33
+ option :beacon_interval, :type => :numeric, :desc => 'Broadcast beacons every N seconds. (default: 5)'
34
+ option :beacon_port, :type => :numeric, :desc => 'Broadcast beacons to this port (default: value of ServiceDirectory.port)'
35
+ option :broadcast_beacons, :type => :boolean, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
32
36
  option :debug, :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
33
37
  option :gc_pause_request, :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
34
38
  option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
@@ -36,26 +40,25 @@ module Protobuf
36
40
  option :worker_port, :type => :numeric, :default => nil, :desc => "Port for 'backend' where workers connect (defaults to port + 1)"
37
41
 
38
42
  def start(app_file)
39
- debug_say 'Configuring the rpc_server process'
40
- @start_aborted = false
43
+ debug_say('Configuring the rpc_server process')
41
44
 
42
45
  configure_logger
43
46
  configure_traps
44
- configure_server_mode
45
- require_protobuf!
47
+ configure_runner_mode
48
+ create_runner
49
+ configure_process_name(app_file)
46
50
  configure_gc
47
51
  configure_deprecation_warnings
48
52
 
49
- run_if_no_abort { require_application!(app_file) }
50
- run_if_no_abort { configure_process_name(app_file) }
51
- run_if_no_abort { start_server! }
53
+ require_application(app_file) unless exit_requested?
54
+ start_server unless exit_requested?
52
55
  rescue => e
53
- say_and_exit!('ERROR: RPC Server failed to start.', e)
56
+ say_and_exit('ERROR: RPC Server failed to start.', e)
54
57
  end
55
58
 
56
59
  desc 'version', 'Print ruby and protoc versions and exit.'
57
60
  def version
58
- say "Ruby Protobuf v#{::Protobuf::VERSION}, protoc v#{::Protobuf::PROTOC_VERSION}"
61
+ say("Ruby Protobuf v#{::Protobuf::VERSION}, protoc v#{::Protobuf::PROTOC_VERSION}")
59
62
  end
60
63
 
61
64
  no_tasks do
@@ -71,7 +74,7 @@ module Protobuf
71
74
 
72
75
  # If we pause during request we don't need to pause in serialization
73
76
  def configure_gc
74
- debug_say 'Configuring gc'
77
+ debug_say('Configuring gc')
75
78
 
76
79
  if defined?(JRUBY_VERSION)
77
80
  # GC.enable/disable are noop's on Jruby
@@ -83,7 +86,7 @@ module Protobuf
83
86
 
84
87
  # Setup the protobuf logger.
85
88
  def configure_logger
86
- debug_say 'Configuring logger'
89
+ debug_say('Configuring logger')
87
90
  ::Protobuf::Logger.configure({ :file => options.log || STDOUT,
88
91
  :level => options.debug? ? ::Logger::DEBUG : options.level })
89
92
 
@@ -94,100 +97,110 @@ module Protobuf
94
97
 
95
98
  # Re-write the $0 var to have a nice process name in ps.
96
99
  def configure_process_name(app_file)
97
- debug_say 'Configuring process name'
98
- $0 = "rpc_server --#{@mode} #{options.host}:#{options.port} #{app_file}"
100
+ debug_say('Configuring process name')
101
+ $0 = "rpc_server --#{@runner_mode} #{options.host}:#{options.port} #{app_file}"
99
102
  end
100
103
 
101
104
  # Configure the mode of the server and the runner class.
102
- def configure_server_mode
103
- debug_say 'Configuring runner mode'
104
- if options.zmq? && ! options.evented? && ! options.socket?
105
- server_zmq!
106
- elsif options.evented? && ! options.zmq? && ! options.socket?
107
- server_evented!
108
- elsif (env_server_type = ENV["PB_SERVER_TYPE"])
109
- case
110
- when env_server_type =~ /zmq/i then
111
- server_zmq!
112
- when env_server_type =~ /socket/i then
113
- server_socket!
114
- when env_server_type =~ /evented/i then
115
- server_evented!
105
+ def configure_runner_mode
106
+ debug_say('Configuring runner mode')
107
+
108
+ if multi_mode?
109
+ say('WARNING: You have provided multiple mode options. Defaulting to socket mode.', :yellow)
110
+ @runner_mode = :socket
111
+ elsif options.zmq?
112
+ @runner_mode = :zmq
113
+ elsif options.evented?
114
+ @runner_mode = :evented
115
+ else
116
+ case server_type = ENV["PB_SERVER_TYPE"]
117
+ when nil, /socket/i
118
+ @runner_mode = :socket
119
+ when /zmq/i
120
+ @runner_mode = :zmq
121
+ when /evented/i
122
+ @runner_mode = :evented
116
123
  else
117
- say "WARNING: You have provided incorrect option 'PB_SERVER_TYPE=#{env_server_type}'. Defaulting to socket mode.", :yellow
118
- server_socket!
124
+ say "WARNING: You have provided incorrect option 'PB_SERVER_TYPE=#{server_type}'. Defaulting to socket mode.", :yellow
125
+ @runner_mode = :socket
119
126
  end
120
- else
121
- say 'WARNING: You have provided multiple mode options. Defaulting to socket mode.', :yellow if multi_mode?
122
- server_socket!
123
127
  end
124
128
  end
125
129
 
126
130
  # Configure signal traps.
127
131
  # TODO add signal handling for hot-reloading the application.
128
132
  def configure_traps
129
- debug_say 'Configuring traps'
130
- [:INT, :QUIT, :TERM].each do |signal|
131
- debug_say "Registering signal trap for #{signal}", :blue
133
+ debug_say('Configuring traps')
134
+
135
+ exit_signals = [:INT, :TERM]
136
+ exit_signals << :QUIT unless defined?(JRUBY_VERSION)
137
+
138
+ exit_signals.each do |signal|
139
+ debug_say("Registering trap for exit signal #{signal}", :blue)
140
+
132
141
  trap(signal) do
133
- ::Protobuf::Logger.info { 'RPC Server shutting down...' }
134
- @start_aborted = true
135
- @runner.stop
136
- ::Protobuf::Logger.info { 'Shutdown complete' }
142
+ @exit_requested = true
143
+ shutdown_server
137
144
  end
138
145
  end
139
146
  end
140
147
 
148
+ # Create the runner for the configured mode
149
+ def create_runner
150
+ debug_say("Creating #{@runner_mode} runner")
151
+ @runner = case @runner_mode
152
+ when :evented
153
+ create_evented_runner
154
+ when :zmq
155
+ create_zmq_runner
156
+ when :socket
157
+ create_socket_runner
158
+ else
159
+ say_and_exit("Unknown runner mode: #{@runner_mode}")
160
+ end
161
+ end
162
+
141
163
  # Say something if we're in debug mode.
142
164
  def debug_say(message, color = :yellow)
143
165
  say(message, color) if options.debug?
144
166
  end
145
167
 
168
+ def exit_requested?
169
+ !!@exit_requested
170
+ end
171
+
146
172
  # Internal helper to determine if the modes are multi-set which is not valid.
147
173
  def multi_mode?
148
- (options.zmq? && (options.evented? || options.socket?)) \
149
- && (options.evented? && (options.evented? || options.socket?)) \
150
- && (options.zmq? && (options.evented? || options.socket?)) \
174
+ [
175
+ options.zmq?,
176
+ options.evented?,
177
+ options.socket?,
178
+ ].count(true) > 1
151
179
  end
152
180
 
153
181
  # Require the application file given, exiting if the file doesn't exist.
154
- def require_application!(app_file)
155
- debug_say 'Requiring app file'
182
+ def require_application(app_file)
183
+ debug_say('Requiring app file')
156
184
  require app_file
157
185
  rescue LoadError => e
158
- say_and_exit!("Failed to load application file #{app_file}", e)
186
+ say_and_exit("Failed to load application file #{app_file}", e)
159
187
  end
160
188
 
161
- # Loads protobuf in the given mode, exiting if somehow the mode is wrong.
162
- def require_protobuf!
163
- require "protobuf/#{@mode}.rb"
164
- rescue LoadError => e
165
- puts e.message, *(e.backtrace)
166
- say_and_exit!("Failed to load protobuf runner #{@mode}", e)
167
- end
189
+ def runner_options
190
+ # Symbolize keys
191
+ opt = options.inject({}) { |h, (k, v)| h[k.to_sym] = v; h }
192
+
193
+ opt[:workers_only] = (!!ENV['PB_WORKERS_ONLY']) || options.workers_only
168
194
 
169
- def run_if_no_abort
170
- yield unless @start_aborted
195
+ opt
171
196
  end
172
197
 
173
- def runner_options
174
- {
175
- :host => options.host,
176
- :port => options.port,
177
- :backlog => options.backlog,
178
- :threshold => options.threshold,
179
- :threads => options.threads,
180
- :worker_port => options.worker_port || (options.port + 1),
181
- :workers_only => !!ENV['PB_WORKERS_ONLY'] || options.workers_only
182
- }
183
- end
184
-
185
- def say_and_exit!(message, exception = nil)
198
+ def say_and_exit(message, exception = nil)
186
199
  message = set_color(message, :red) if ::Protobuf::Logger.file == STDOUT
187
200
 
188
201
  ::Protobuf::Logger.error { message }
189
202
  if exception
190
- $stderr.puts "[#{exception.class.name}] #{exception.message}"
203
+ $stderr.puts "[#{exception.class.name}] #{exception.message}"
191
204
  $stderr.puts exception.backtrace.join("\n")
192
205
 
193
206
  ::Protobuf::Logger.error { "[#{exception.class.name}] #{exception.message}" }
@@ -197,35 +210,42 @@ module Protobuf
197
210
  exit(1)
198
211
  end
199
212
 
200
- def server_evented!
201
- @mode = :evented
202
- @runner = ::Protobuf::Rpc::EventedRunner
213
+ def create_evented_runner
214
+ require 'protobuf/evented'
215
+
216
+ @runner = ::Protobuf::Rpc::EventedRunner.new(runner_options)
217
+ end
218
+
219
+ def create_socket_runner
220
+ require 'protobuf/socket'
221
+
222
+ @runner = ::Protobuf::Rpc::SocketRunner.new(runner_options)
203
223
  end
204
224
 
205
- def server_socket!
206
- @mode = :socket
207
- @runner = ::Protobuf::Rpc::SocketRunner
225
+ def create_zmq_runner
226
+ require 'protobuf/zmq'
227
+
228
+ @runner = ::Protobuf::Rpc::ZmqRunner.new(runner_options)
208
229
  end
209
230
 
210
- def server_zmq!
211
- @mode = :zmq
212
- @runner = ::Protobuf::Rpc::ZmqRunner
231
+ def shutdown_server
232
+ ::Protobuf::Logger.info { 'RPC Server shutting down...' }
233
+ @runner.try(:stop)
234
+ ::Protobuf::Rpc::ServiceDirectory.instance.stop
235
+ ::Protobuf::Logger.info { 'Shutdown complete' }
213
236
  end
214
237
 
215
238
  # Start the runner and log the relevant options.
216
- def start_server!
217
- @runner.register_signals
239
+ def start_server
240
+ debug_say('Running server')
218
241
 
219
- debug_say 'Invoking server start'
220
- @runner.run(runner_options) do
221
- ::Protobuf::Logger.info {
222
- "pid #{::Process.pid} -- #{@mode} RPC Server listening at #{options.host}:#{options.port}"
242
+ @runner.run do
243
+ ::Protobuf::Logger.info {
244
+ "pid #{::Process.pid} -- #{@runner_mode} RPC Server listening at #{options.host}:#{options.port}"
223
245
  }
224
246
  end
225
247
  end
226
-
227
248
  end
228
-
229
249
  end
230
250
  end
231
251
 
@@ -18,9 +18,8 @@ module Protobuf
18
18
  end
19
19
 
20
20
  def decode(bytes)
21
- bytes_to_decode = bytes.dup
22
- bytes_to_decode.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
23
- bytes_to_decode
21
+ bytes.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
22
+ bytes
24
23
  end
25
24
 
26
25
  def define_setter
@@ -47,12 +46,11 @@ module Protobuf
47
46
  end
48
47
 
49
48
  def encode(value)
50
- value_to_encode = value.dup
51
- value_to_encode = value.serialize_to_string if value.is_a?(::Protobuf::Message)
52
- value_to_encode.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
49
+ value = value.serialize_to_string if value.is_a?(::Protobuf::Message)
50
+ value.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
53
51
 
54
- string_size = ::Protobuf::Field::VarintField.encode(value_to_encode.size)
55
- string_size << value_to_encode
52
+ string_size = ::Protobuf::Field::VarintField.encode(value.size)
53
+ string_size << value
56
54
  end
57
55
 
58
56
  def wire_type