protobuf 2.8.0.beta2-java → 2.8.0.beta3-java

Sign up to get free protection for your applications and to get access to all the features.
@@ -3,7 +3,16 @@ require 'socket'
3
3
  require 'pp'
4
4
  require 'stringio'
5
5
  require 'active_support/core_ext/object/blank'
6
- require 'active_support/core_ext/object/try'
6
+ require 'active_support/version'
7
+
8
+ if ActiveSupport::VERSION::MAJOR > 2
9
+ require 'active_support/core_ext/object/try'
10
+ else
11
+ require 'active_support/core_ext/module/delegation'
12
+ require 'active_support/core_ext/kernel/reporting'
13
+ require 'active_support/core_ext/try'
14
+ end
15
+
7
16
  require 'active_support/inflector'
8
17
  require 'active_support/json'
9
18
 
@@ -4,10 +4,6 @@ module Protobuf
4
4
  module Field
5
5
  class FloatField < BaseField
6
6
  def self.default; 0.0; end
7
- def self.max; 1.0/0; end
8
- def self.min; -1.0/0; end
9
- def max; 1.0/0; end
10
- def min; -1.0/0; end
11
7
 
12
8
  def wire_type
13
9
  WireType::FIXED32
@@ -22,7 +18,7 @@ module Protobuf
22
18
  end
23
19
 
24
20
  def acceptable?(val)
25
- (val > min || val < max) rescue false
21
+ val.respond_to?(:to_f)
26
22
  end
27
23
  end
28
24
  end
@@ -19,6 +19,13 @@ module Protobuf
19
19
 
20
20
  CLIENT_RETRIES = (ENV['PB_CLIENT_RETRIES'] || 3)
21
21
 
22
+ ##
23
+ # Class Methods
24
+ #
25
+ def self.zmq_context
26
+ @zmq_context ||= ZMQ::Context.new
27
+ end
28
+
22
29
  ##
23
30
  # Instance methods
24
31
  #
@@ -47,7 +54,6 @@ module Protobuf
47
54
 
48
55
  def close_connection
49
56
  socket_close
50
- zmq_context_terminate
51
57
  end
52
58
 
53
59
  # Establish a request socket connection to the remote rpc_server.
@@ -106,7 +112,7 @@ module Protobuf
106
112
  read_response
107
113
  return
108
114
  else
109
- close_connection
115
+ socket_close
110
116
  end
111
117
  end
112
118
 
@@ -171,16 +177,7 @@ module Protobuf
171
177
  # an exit block to ensure the context is terminated correctly.
172
178
  #
173
179
  def zmq_context
174
- @zmq_context ||= ::ZMQ::Context.new
175
- end
176
-
177
- # Terminate the zmq_context (if any).
178
- #
179
- def zmq_context_terminate
180
- log_debug { sign_message("Terminating ZMQ Context") }
181
- @zmq_context.try(:terminate)
182
- @zmq_context = nil
183
- log_debug { sign_message("ZMQ Context terminated") }
180
+ self.class.zmq_context
184
181
  end
185
182
 
186
183
  def zmq_error_check(return_code, source)
@@ -1,4 +1,3 @@
1
-
2
1
  module Protobuf
3
2
  module Rpc
4
3
  module Zmq
@@ -11,31 +10,24 @@ module Protobuf
11
10
  init_zmq_context
12
11
  init_backend_socket
13
12
  init_frontend_socket
14
- init_shutdown_socket
15
13
  init_poller
16
14
  rescue
17
15
  teardown
18
16
  raise
19
17
  end
20
18
 
21
- def join
22
- @thread.try(:join)
23
- end
24
-
25
19
  def run
26
20
  @idle_workers = []
27
21
 
28
- catch(:shutdown) do
29
- while @poller.poll > 0
30
- @poller.readables.each do |readable|
31
- case readable
32
- when @frontend_socket
33
- process_frontend
34
- when @backend_socket
35
- process_backend
36
- when @shutdown_socket
37
- throw :shutdown
38
- end
22
+ while running?
23
+ break if @poller.poll(500) < 0
24
+
25
+ @poller.readables.each do |readable|
26
+ case readable
27
+ when @frontend_socket
28
+ process_frontend
29
+ when @backend_socket
30
+ process_backend
39
31
  end
40
32
  end
41
33
  end
@@ -43,23 +35,8 @@ module Protobuf
43
35
  teardown
44
36
  end
45
37
 
46
- def start
47
- log_debug { sign_message("starting broker") }
48
-
49
- @thread = Thread.new { self.run }
50
-
51
- self
52
- end
53
-
54
- def shutdown_uri
55
- "inproc://#{object_id}"
56
- end
57
-
58
- def signal_shutdown
59
- socket = @zmq_context.socket(ZMQ::PAIR)
60
- zmq_error_check(socket.connect(shutdown_uri))
61
- zmq_error_check(socket.send_string ".")
62
- zmq_error_check(socket.close)
38
+ def running?
39
+ @server.running? || @server.workers.any?
63
40
  end
64
41
 
65
42
  private
@@ -78,12 +55,6 @@ module Protobuf
78
55
  @poller = ZMQ::Poller.new
79
56
  @poller.register_readable(@frontend_socket)
80
57
  @poller.register_readable(@backend_socket)
81
- @poller.register_readable(@shutdown_socket)
82
- end
83
-
84
- def init_shutdown_socket
85
- @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
86
- zmq_error_check(@shutdown_socket.bind(shutdown_uri))
87
58
  end
88
59
 
89
60
  def init_zmq_context
@@ -108,21 +79,20 @@ module Protobuf
108
79
  end
109
80
 
110
81
  def read_from_backend
111
- [].tap do |frames|
112
- zmq_error_check(@backend_socket.recv_strings(frames))
113
- end
82
+ frames = []
83
+ zmq_error_check(@backend_socket.recv_strings(frames))
84
+ frames
114
85
  end
115
86
 
116
87
  def read_from_frontend
117
- [].tap do |frames|
118
- zmq_error_check(@frontend_socket.recv_strings(frames))
119
- end
88
+ frames = []
89
+ zmq_error_check(@frontend_socket.recv_strings(frames))
90
+ frames
120
91
  end
121
92
 
122
93
  def teardown
123
94
  @frontend_socket.try(:close)
124
95
  @backend_socket.try(:close)
125
- @shutdown_socket.try(:close)
126
96
  @zmq_context.try(:terminate)
127
97
  end
128
98
 
@@ -15,7 +15,7 @@ module Protobuf
15
15
  :broadcast_beacons => false
16
16
  }
17
17
 
18
- attr_accessor :options
18
+ attr_accessor :options, :workers
19
19
 
20
20
  def initialize(options)
21
21
  @options = DEFAULT_OPTIONS.merge(options)
@@ -23,12 +23,16 @@ module Protobuf
23
23
 
24
24
  init_zmq_context
25
25
  init_beacon_socket if broadcast_beacons?
26
- init_shutdown_socket
26
+ init_shutdown_pipe
27
27
  rescue
28
28
  teardown
29
29
  raise
30
30
  end
31
31
 
32
+ def add_worker
33
+ @total_workers = total_workers + 1
34
+ end
35
+
32
36
  def backend_ip
33
37
  frontend_ip
34
38
  end
@@ -120,7 +124,7 @@ module Protobuf
120
124
  end
121
125
 
122
126
  def maintenance_timeout
123
- 1_000 * (next_maintenance - Time.now.to_i)
127
+ next_maintenance - Time.now.to_i
124
128
  end
125
129
 
126
130
  def next_maintenance
@@ -131,7 +135,7 @@ module Protobuf
131
135
  end
132
136
 
133
137
  def minimum_timeout
134
- 100
138
+ 0.1
135
139
  end
136
140
 
137
141
  def next_beacon
@@ -173,8 +177,8 @@ module Protobuf
173
177
  start_missing_workers
174
178
  wait_for_shutdown_signal
175
179
  broadcast_flatline if broadcast_beacons?
176
- stop_workers
177
- stop_broker unless brokerless?
180
+ Thread.pass until reap_dead_workers.empty?
181
+ @broker.join unless brokerless?
178
182
  ensure
179
183
  @running = false
180
184
  teardown
@@ -184,21 +188,6 @@ module Protobuf
184
188
  !!@running
185
189
  end
186
190
 
187
- def shutdown_uri
188
- "inproc://#{object_id}"
189
- end
190
-
191
- def signal_shutdown
192
- socket = @zmq_context.socket ZMQ::PAIR
193
- zmq_error_check(socket.connect shutdown_uri)
194
- zmq_error_check(socket.send_string ".")
195
- zmq_error_check(socket.close)
196
- end
197
-
198
- def start_broker
199
- @broker = ::Protobuf::Rpc::Zmq::Broker.new(self).start
200
- end
201
-
202
191
  def start_missing_workers
203
192
  missing_workers = total_workers - @workers.size
204
193
 
@@ -208,26 +197,14 @@ module Protobuf
208
197
  end
209
198
  end
210
199
 
211
- def start_worker
212
- @workers << ::Protobuf::Rpc::Zmq::Worker.new(self).start
213
- end
214
-
215
200
  def stop
216
- signal_shutdown
217
- end
218
-
219
- def stop_broker
220
- @broker.signal_shutdown
221
- @broker.join
222
- end
223
-
224
- def stop_workers
225
- @workers.each(&:signal_shutdown)
226
- Thread.pass until reap_dead_workers.empty?
201
+ @running = false
202
+ @shutdown_w.write('.')
227
203
  end
228
204
 
229
205
  def teardown
230
- @shutdown_socket.try(:close)
206
+ @shutdown_r.try(:close)
207
+ @shutdown_w.try(:close)
231
208
  @beacon_socket.try(:close)
232
209
  @zmq_context.try(:terminate)
233
210
  @last_reaping = @last_beacon = @timeout = nil
@@ -260,12 +237,9 @@ module Protobuf
260
237
  end
261
238
 
262
239
  def wait_for_shutdown_signal
263
- poller = ZMQ::Poller.new
264
- poller.register_readable(@shutdown_socket)
240
+ loop do
241
+ break if IO.select([@shutdown_r], nil, nil, timeout)
265
242
 
266
- # If the poller returns 1, a shutdown signal has been received.
267
- # If the poller returns -1, something went wrong.
268
- while poller.poll(timeout) === 0
269
243
  if reap_dead_workers?
270
244
  reap_dead_workers
271
245
  start_missing_workers
@@ -283,14 +257,31 @@ module Protobuf
283
257
  @beacon_socket.connect(beacon_ip, beacon_port)
284
258
  end
285
259
 
286
- def init_shutdown_socket
287
- @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
288
- zmq_error_check(@shutdown_socket.bind shutdown_uri)
260
+ def init_shutdown_pipe
261
+ @shutdown_r, @shutdown_w = IO.pipe
289
262
  end
290
263
 
291
264
  def init_zmq_context
292
265
  @zmq_context = ZMQ::Context.new
293
266
  end
267
+
268
+ def start_broker
269
+ @broker = Thread.new(self) do |server|
270
+ ::Protobuf::Rpc::Zmq::Broker.new(server).run
271
+ end
272
+ end
273
+
274
+ def start_worker
275
+ @workers << Thread.new(self) do |server|
276
+ begin
277
+ ::Protobuf::Rpc::Zmq::Worker.new(server).run
278
+ rescue => e
279
+ message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
280
+ $stderr.puts(message)
281
+ log_error { message }
282
+ end
283
+ end
284
+ end
294
285
  end
295
286
  end
296
287
  end
@@ -13,9 +13,9 @@ module Protobuf
13
13
  #
14
14
  def initialize(server)
15
15
  @server = server
16
+
16
17
  init_zmq_context
17
18
  init_backend_socket
18
- init_shutdown_socket
19
19
  rescue
20
20
  teardown
21
21
  raise
@@ -24,14 +24,6 @@ module Protobuf
24
24
  ##
25
25
  # Instance Methods
26
26
  #
27
- def alive?
28
- @thread.try(:alive?) || false
29
- end
30
-
31
- def join
32
- @thread.try(:join)
33
- end
34
-
35
27
  def process_request
36
28
  @client_address, empty, @request_data = read_from_backend
37
29
 
@@ -41,12 +33,6 @@ module Protobuf
41
33
  end
42
34
  end
43
35
 
44
- def read_from_backend
45
- [].tap do |frames|
46
- zmq_error_check(@backend_socket.recv_strings(frames))
47
- end
48
- end
49
-
50
36
  def run
51
37
  poller = ::ZMQ::Poller.new
52
38
  poller.register_readable(@backend_socket)
@@ -55,23 +41,22 @@ module Protobuf
55
41
  # Send request to broker telling it we are ready
56
42
  write_to_backend([::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE])
57
43
 
58
- catch(:shutdown) do
59
- while poller.poll > 0
60
- poller.readables.each do |readable|
61
- case readable
62
- when @backend_socket
63
- initialize_request!
64
- process_request
65
- when @shutdown_socket
66
- throw :shutdown
67
- end
68
- end
44
+ while running?
45
+ break if poller.poll(500) < 0
46
+
47
+ if poller.readables.any?
48
+ initialize_request!
49
+ process_request
69
50
  end
70
51
  end
71
52
  ensure
72
53
  teardown
73
54
  end
74
55
 
56
+ def running?
57
+ @server.running?
58
+ end
59
+
75
60
  def send_data
76
61
  data = @response.serialize_to_string
77
62
 
@@ -80,56 +65,31 @@ module Protobuf
80
65
  write_to_backend([@client_address, "", data])
81
66
  end
82
67
 
83
- def shutdown_uri
84
- "inproc://#{object_id}"
85
- end
68
+ private
86
69
 
87
- def signal_shutdown
88
- socket = @zmq_context.socket ZMQ::PAIR
89
- zmq_error_check(socket.connect(shutdown_uri))
90
- zmq_error_check(socket.send_string("."))
91
- zmq_error_check(socket.close)
70
+ def init_zmq_context
71
+ @zmq_context = ZMQ::Context.new
92
72
  end
93
73
 
94
- def start
95
- @thread = Thread.new do
96
- begin
97
- self.run
98
- rescue => e
99
- message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
100
- $stderr.puts(message)
101
- log_error { message }
102
- end
103
- end
74
+ def init_backend_socket
75
+ @backend_socket = @zmq_context.socket(ZMQ::REQ)
76
+ zmq_error_check(@backend_socket.connect(@server.backend_uri))
77
+ end
104
78
 
105
- self
79
+ def read_from_backend
80
+ frames = []
81
+ zmq_error_check(@backend_socket.recv_strings(frames))
82
+ frames
106
83
  end
107
84
 
108
85
  def teardown
109
86
  @backend_socket.try(:close)
110
- @shutdown_socket.try(:close)
111
87
  @zmq_context.try(:terminate)
112
88
  end
113
89
 
114
90
  def write_to_backend(frames)
115
91
  zmq_error_check(@backend_socket.send_strings(frames))
116
92
  end
117
-
118
- private
119
-
120
- def init_zmq_context
121
- @zmq_context = ZMQ::Context.new
122
- end
123
-
124
- def init_backend_socket
125
- @backend_socket = @zmq_context.socket(ZMQ::REQ)
126
- zmq_error_check(@backend_socket.connect(@server.backend_uri))
127
- end
128
-
129
- def init_shutdown_socket
130
- @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
131
- zmq_error_check(@shutdown_socket.bind(shutdown_uri))
132
- end
133
93
  end
134
94
  end
135
95
  end
@@ -36,9 +36,8 @@ module Protobuf
36
36
 
37
37
  def register_signals
38
38
  trap(:TTIN) do
39
- log_info { "TTIN received: Starting new worker" }
40
- @server.start_worker
41
- log_info { "Worker count : #{::Protobuf::Rpc::Zmq::Server.threads.size}" }
39
+ @server.add_worker
40
+ log_info { "Increased worker size to: #{@server.total_workers}" }
42
41
  end
43
42
  end
44
43
  end
@@ -183,7 +183,8 @@ module Protobuf
183
183
 
184
184
  def run
185
185
  loop do
186
- process_beacon(wait_for_beacon)
186
+ beacon = wait_for_beacon
187
+ process_beacon(beacon)
187
188
  remove_expired_listings
188
189
  end
189
190
  rescue => e
@@ -194,12 +195,13 @@ module Protobuf
194
195
  def wait_for_beacon
195
196
  data, addr = @socket.recvfrom(2048)
196
197
 
197
- ::Protobuf::Rpc::DynamicDiscovery::Beacon.new.tap do |beacon|
198
- beacon.parse_from_string(data) rescue nil
198
+ beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
199
+ beacon.parse_from_string(data)
199
200
 
200
- # Favor the address captured by the socket
201
- beacon.try(:server).try(:address=, addr[3])
202
- end
201
+ # Favor the address captured by the socket
202
+ beacon.try(:server).try(:address=, addr[3])
203
+
204
+ beacon
203
205
  end
204
206
  end
205
207
  end
@@ -1,4 +1,4 @@
1
1
  module Protobuf
2
- VERSION = '2.8.0.beta2'
2
+ VERSION = '2.8.0.beta3'
3
3
  PROTOC_VERSION = '2.4.1'
4
4
  end
@@ -1,4 +1,5 @@
1
1
  require 'spec_helper'
2
+
2
3
  require 'spec/support/test/resource_service'
3
4
  require 'protobuf/rpc/service_directory'
4
5
 
@@ -13,11 +14,13 @@ describe 'Functional ZMQ Client' do
13
14
  }}
14
15
 
15
16
  let(:server) { ::Protobuf::Rpc::Zmq::Server.new(options) }
16
- let(:server_thread) { Thread.new(server) { |server| server.run } }
17
+ let!(:server_thread) { Thread.new(server) { |server| server.run } }
17
18
 
18
- before do
19
+ before(:all) do
19
20
  load "protobuf/zmq.rb"
20
- server_thread.abort_on_exception = true
21
+ end
22
+
23
+ before do
21
24
  Thread.pass until server.running?
22
25
  end
23
26
 
@@ -43,17 +46,36 @@ describe 'Functional ZMQ Client' do
43
46
  }.to_not raise_error
44
47
  end
45
48
 
49
+ it 'runs under heavy load' do
50
+ 100.times do |x|
51
+ 50.times.map do |y|
52
+ Thread.new do
53
+ client = ::Test::ResourceService.client
54
+
55
+ client.find(:name => 'Test Name', :active => true) do |c|
56
+ c.on_success do |succ|
57
+ succ.name.should eq("Test Name")
58
+ succ.status.should eq(::Test::StatusType::ENABLED)
59
+ end
60
+
61
+ c.on_failure do |err|
62
+ raise err.inspect
63
+ end
64
+ end
65
+ end
66
+ end.each(&:join)
67
+ end
68
+ end
69
+
46
70
  context 'when a message is malformed' do
47
71
  it 'calls the on_failure callback' do
48
72
  error = nil
49
- StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
50
- request = ::Test::ResourceFindRequest.new(:active => true)
51
- client = ::Test::ResourceService.client
73
+ request = ::Test::ResourceFindRequest.new(:active => true)
74
+ client = ::Test::ResourceService.client
52
75
 
53
- client.find(request) do |c|
54
- c.on_success { raise "shouldn't pass" }
55
- c.on_failure {|e| error = e }
56
- end
76
+ client.find(request) do |c|
77
+ c.on_success { raise "shouldn't pass" }
78
+ c.on_failure {|e| error = e }
57
79
  end
58
80
  error.message.should match(/name.*required/)
59
81
  end
@@ -62,14 +84,12 @@ describe 'Functional ZMQ Client' do
62
84
  context 'when the request type is wrong' do
63
85
  it 'calls the on_failure callback' do
64
86
  error = nil
65
- StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
66
- request = ::Test::Resource.new(:name => 'Test Name')
67
- client = ::Test::ResourceService.client
87
+ request = ::Test::Resource.new(:name => 'Test Name')
88
+ client = ::Test::ResourceService.client
68
89
 
69
- client.find(request) do |c|
70
- c.on_success { raise "shouldn't pass" }
71
- c.on_failure {|e| error = e}
72
- end
90
+ client.find(request) do |c|
91
+ c.on_success { raise "shouldn't pass" }
92
+ c.on_failure {|e| error = e}
73
93
  end
74
94
  error.message.should match(/expected request.*ResourceFindRequest.*Resource instead/i)
75
95
  end
@@ -78,13 +98,11 @@ describe 'Functional ZMQ Client' do
78
98
  context 'when the server takes too long to respond' do
79
99
  it 'responds with a timeout error' do
80
100
  error = nil
81
- StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
82
- client = ::Test::ResourceService.client(:timeout => 1)
101
+ client = ::Test::ResourceService.client(:timeout => 1)
83
102
 
84
- client.find_with_sleep(:sleep => 2) do |c|
85
- c.on_success { raise "shouldn't pass" }
86
- c.on_failure { |e| error = e }
87
- end
103
+ client.find_with_sleep(:sleep => 2) do |c|
104
+ c.on_success { raise "shouldn't pass" }
105
+ c.on_failure { |e| error = e }
88
106
  end
89
107
  error.message.should match(/The server took longer than 1 seconds to respond/i)
90
108
  end
@@ -32,11 +32,6 @@ describe Protobuf::Rpc::Zmq::Server do
32
32
  end
33
33
 
34
34
  describe '.stop' do
35
- it 'signals shutdown' do
36
- subject.should_receive(:signal_shutdown)
37
- subject.stop
38
- end
39
-
40
35
  it 'sets running to false' do
41
36
  subject.instance_variable_set(:@workers, [])
42
37
  subject.stop
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: protobuf
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease: 6
5
- version: 2.8.0.beta2
5
+ version: 2.8.0.beta3
6
6
  platform: java
7
7
  authors:
8
8
  - BJ Neilsen
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2013-06-18 00:00:00.000000000 Z
13
+ date: 2013-06-20 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activesupport