protobuf 2.8.10 → 2.8.11

Sign up to get free protection for your applications and to get access to all the features.
data/CHANGES.md CHANGED
@@ -1,3 +1,11 @@
1
+ 2.8.11
2
+ ---------
3
+
4
+ - Default ZMQ server to use inproc protocol instead of tcp (zero-copy between server-broker-worker)
5
+ - Add 'broadcast_busy' functionality that removes server from cluster if the workers are full
6
+ - Add cli option for --no-zmq_inproc
7
+ - Add cli option for --broadcast_busy
8
+
1
9
  2.8.10
2
10
  ---------
3
11
 
@@ -32,11 +32,13 @@ module Protobuf
32
32
  option :beacon_interval, :type => :numeric, :desc => 'Broadcast beacons every N seconds. (default: 5)'
33
33
  option :beacon_port, :type => :numeric, :desc => 'Broadcast beacons to this port (default: value of ServiceDirectory.port)'
34
34
  option :broadcast_beacons, :type => :boolean, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
35
+ option :broadcast_busy, :type => :boolean, :default => false, :desc => 'Remove busy nodes from cluster when all workers are busy (Currently only available with ZeroMQ).'
35
36
  option :debug, :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
36
37
  option :gc_pause_request, :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
37
38
  option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
38
39
  option :workers_only, :type => :boolean, :default => false, :desc => "Starts process with only workers (no broker/frontend is started) only relevant for Zmq Server"
39
40
  option :worker_port, :type => :numeric, :default => nil, :desc => "Port for 'backend' where workers connect (defaults to port + 1)"
41
+ option :zmq_inproc, :type => :boolean, :default => true, :desc => 'Use inproc protocol for zmq Server/Broker/Worker'
40
42
 
41
43
  def start(app_file)
42
44
  debug_say('Configuring the rpc_server process')
@@ -64,7 +64,15 @@ module Protobuf
64
64
  end
65
65
 
66
66
  def init_zmq_context
67
- @zmq_context = ZMQ::Context.new
67
+ if inproc?
68
+ @zmq_context = @server.zmq_context
69
+ else
70
+ @zmq_context = ZMQ::Context.new
71
+ end
72
+ end
73
+
74
+ def inproc?
75
+ !!@server.try(:inproc?)
68
76
  end
69
77
 
70
78
  def process_backend
@@ -12,10 +12,13 @@ module Protobuf
12
12
 
13
13
  DEFAULT_OPTIONS = {
14
14
  :beacon_interval => 5,
15
- :broadcast_beacons => false
15
+ :broadcast_beacons => false,
16
+ :broadcast_busy => false,
17
+ :zmq_inproc => true,
16
18
  }
17
19
 
18
20
  attr_accessor :options, :workers
21
+ attr_reader :zmq_context
19
22
 
20
23
  def initialize(options)
21
24
  @options = DEFAULT_OPTIONS.merge(options)
@@ -33,12 +36,20 @@ module Protobuf
33
36
  @total_workers = total_workers + 1
34
37
  end
35
38
 
39
+ def all_workers_busy?
40
+ workers.all? { |thread| !!thread.thread_variable_get(:busy) }
41
+ end
42
+
36
43
  def backend_port
37
44
  options[:worker_port] || frontend_port + 1
38
45
  end
39
46
 
40
47
  def backend_uri
41
- "tcp://#{backend_ip}:#{backend_port}"
48
+ if inproc?
49
+ "inproc://#{backend_ip}:#{backend_port}"
50
+ else
51
+ "tcp://#{backend_ip}:#{backend_port}"
52
+ end
42
53
  end
43
54
 
44
55
  def beacon_interval
@@ -99,6 +110,10 @@ module Protobuf
99
110
  !!options[:workers_only]
100
111
  end
101
112
 
113
+ def busy_worker_count
114
+ workers.count { |thread| !!thread.thread_variable_get(:busy) }
115
+ end
116
+
102
117
  def frontend_ip
103
118
  @frontend_ip ||= resolve_ip(options[:host])
104
119
  end
@@ -112,6 +127,10 @@ module Protobuf
112
127
  "tcp://#{frontend_ip}:#{frontend_port}"
113
128
  end
114
129
 
130
+ def inproc?
131
+ !!self.options[:zmq_inproc]
132
+ end
133
+
115
134
  def maintenance_timeout
116
135
  next_maintenance - Time.now.to_i
117
136
  end
@@ -236,7 +255,14 @@ module Protobuf
236
255
  start_missing_workers
237
256
  end
238
257
 
239
- broadcast_heartbeat if broadcast_heartbeat?
258
+ if broadcast_heartbeat?
259
+ if all_workers_busy? && options[:broadcast_busy]
260
+ broadcast_flatline
261
+ else
262
+ broadcast_heartbeat
263
+ end
264
+ end
265
+
240
266
  end
241
267
  end
242
268
 
@@ -51,8 +51,10 @@ module Protobuf
51
51
  break if rc == -1
52
52
 
53
53
  if rc > 0
54
+ ::Thread.current.thread_variable_set(:busy, true)
54
55
  initialize_request!
55
56
  process_request
57
+ ::Thread.current.thread_variable_set(:busy, false)
56
58
  end
57
59
  end
58
60
  ensure
@@ -74,7 +76,11 @@ module Protobuf
74
76
  private
75
77
 
76
78
  def init_zmq_context
77
- @zmq_context = ZMQ::Context.new
79
+ if inproc?
80
+ @zmq_context = @server.zmq_context
81
+ else
82
+ @zmq_context = ZMQ::Context.new
83
+ end
78
84
  end
79
85
 
80
86
  def init_backend_socket
@@ -82,6 +88,10 @@ module Protobuf
82
88
  zmq_error_check(@backend_socket.connect(@server.backend_uri))
83
89
  end
84
90
 
91
+ def inproc?
92
+ !!@server.try(:inproc?)
93
+ end
94
+
85
95
  def read_from_backend
86
96
  frames = []
87
97
  zmq_error_check(@backend_socket.recv_strings(frames))
@@ -43,6 +43,7 @@ module Protobuf
43
43
 
44
44
  trap(:TTOU) do
45
45
  log_info { "Current worker size: #{@server.workers.size}" }
46
+ log_info { "Current worker size: #{@server.busy_worker_count}" }
46
47
  end
47
48
  end
48
49
  end
@@ -1,4 +1,4 @@
1
1
  module Protobuf
2
- VERSION = '2.8.10'
2
+ VERSION = '2.8.11'
3
3
  PROTOC_VERSION = '2.5.0'
4
4
  end
metadata CHANGED
@@ -1,7 +1,8 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: protobuf
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.8.10
4
+ version: 2.8.11
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - BJ Neilsen
@@ -10,160 +11,182 @@ authors:
10
11
  autorequire:
11
12
  bindir: bin
12
13
  cert_chain: []
13
- date: 2014-02-06 00:00:00.000000000 Z
14
+ date: 2014-02-10 00:00:00.000000000 Z
14
15
  dependencies:
15
16
  - !ruby/object:Gem::Dependency
16
17
  name: activesupport
17
18
  requirement: !ruby/object:Gem::Requirement
19
+ none: false
18
20
  requirements:
19
- - - '>='
21
+ - - ! '>='
20
22
  - !ruby/object:Gem::Version
21
23
  version: '0'
22
24
  type: :runtime
23
25
  prerelease: false
24
26
  version_requirements: !ruby/object:Gem::Requirement
27
+ none: false
25
28
  requirements:
26
- - - '>='
29
+ - - ! '>='
27
30
  - !ruby/object:Gem::Version
28
31
  version: '0'
29
32
  - !ruby/object:Gem::Dependency
30
33
  name: multi_json
31
34
  requirement: !ruby/object:Gem::Requirement
35
+ none: false
32
36
  requirements:
33
- - - '>='
37
+ - - ! '>='
34
38
  - !ruby/object:Gem::Version
35
39
  version: '0'
36
40
  type: :runtime
37
41
  prerelease: false
38
42
  version_requirements: !ruby/object:Gem::Requirement
43
+ none: false
39
44
  requirements:
40
- - - '>='
45
+ - - ! '>='
41
46
  - !ruby/object:Gem::Version
42
47
  version: '0'
43
48
  - !ruby/object:Gem::Dependency
44
49
  name: thor
45
50
  requirement: !ruby/object:Gem::Requirement
51
+ none: false
46
52
  requirements:
47
- - - '>='
53
+ - - ! '>='
48
54
  - !ruby/object:Gem::Version
49
55
  version: '0'
50
56
  type: :runtime
51
57
  prerelease: false
52
58
  version_requirements: !ruby/object:Gem::Requirement
59
+ none: false
53
60
  requirements:
54
- - - '>='
61
+ - - ! '>='
55
62
  - !ruby/object:Gem::Version
56
63
  version: '0'
57
64
  - !ruby/object:Gem::Dependency
58
65
  name: eventmachine
59
66
  requirement: !ruby/object:Gem::Requirement
67
+ none: false
60
68
  requirements:
61
- - - '>='
69
+ - - ! '>='
62
70
  - !ruby/object:Gem::Version
63
71
  version: '0'
64
72
  type: :development
65
73
  prerelease: false
66
74
  version_requirements: !ruby/object:Gem::Requirement
75
+ none: false
67
76
  requirements:
68
- - - '>='
77
+ - - ! '>='
69
78
  - !ruby/object:Gem::Version
70
79
  version: '0'
71
80
  - !ruby/object:Gem::Dependency
72
81
  name: ffi-rzmq
73
82
  requirement: !ruby/object:Gem::Requirement
83
+ none: false
74
84
  requirements:
75
- - - '>='
85
+ - - ! '>='
76
86
  - !ruby/object:Gem::Version
77
87
  version: '0'
78
88
  type: :development
79
89
  prerelease: false
80
90
  version_requirements: !ruby/object:Gem::Requirement
91
+ none: false
81
92
  requirements:
82
- - - '>='
93
+ - - ! '>='
83
94
  - !ruby/object:Gem::Version
84
95
  version: '0'
85
96
  - !ruby/object:Gem::Dependency
86
97
  name: pry-nav
87
98
  requirement: !ruby/object:Gem::Requirement
99
+ none: false
88
100
  requirements:
89
- - - '>='
101
+ - - ! '>='
90
102
  - !ruby/object:Gem::Version
91
103
  version: '0'
92
104
  type: :development
93
105
  prerelease: false
94
106
  version_requirements: !ruby/object:Gem::Requirement
107
+ none: false
95
108
  requirements:
96
- - - '>='
109
+ - - ! '>='
97
110
  - !ruby/object:Gem::Version
98
111
  version: '0'
99
112
  - !ruby/object:Gem::Dependency
100
113
  name: rake
101
114
  requirement: !ruby/object:Gem::Requirement
115
+ none: false
102
116
  requirements:
103
- - - '>='
117
+ - - ! '>='
104
118
  - !ruby/object:Gem::Version
105
119
  version: '0'
106
120
  type: :development
107
121
  prerelease: false
108
122
  version_requirements: !ruby/object:Gem::Requirement
123
+ none: false
109
124
  requirements:
110
- - - '>='
125
+ - - ! '>='
111
126
  - !ruby/object:Gem::Version
112
127
  version: '0'
113
128
  - !ruby/object:Gem::Dependency
114
129
  name: rspec
115
130
  requirement: !ruby/object:Gem::Requirement
131
+ none: false
116
132
  requirements:
117
- - - '>='
133
+ - - ! '>='
118
134
  - !ruby/object:Gem::Version
119
135
  version: '0'
120
136
  type: :development
121
137
  prerelease: false
122
138
  version_requirements: !ruby/object:Gem::Requirement
139
+ none: false
123
140
  requirements:
124
- - - '>='
141
+ - - ! '>='
125
142
  - !ruby/object:Gem::Version
126
143
  version: '0'
127
144
  - !ruby/object:Gem::Dependency
128
145
  name: simplecov
129
146
  requirement: !ruby/object:Gem::Requirement
147
+ none: false
130
148
  requirements:
131
- - - '>='
149
+ - - ! '>='
132
150
  - !ruby/object:Gem::Version
133
151
  version: '0'
134
152
  type: :development
135
153
  prerelease: false
136
154
  version_requirements: !ruby/object:Gem::Requirement
155
+ none: false
137
156
  requirements:
138
- - - '>='
157
+ - - ! '>='
139
158
  - !ruby/object:Gem::Version
140
159
  version: '0'
141
160
  - !ruby/object:Gem::Dependency
142
161
  name: yard
143
162
  requirement: !ruby/object:Gem::Requirement
163
+ none: false
144
164
  requirements:
145
- - - '>='
165
+ - - ! '>='
146
166
  - !ruby/object:Gem::Version
147
167
  version: '0'
148
168
  type: :development
149
169
  prerelease: false
150
170
  version_requirements: !ruby/object:Gem::Requirement
171
+ none: false
151
172
  requirements:
152
- - - '>='
173
+ - - ! '>='
153
174
  - !ruby/object:Gem::Version
154
175
  version: '0'
155
176
  - !ruby/object:Gem::Dependency
156
177
  name: timecop
157
178
  requirement: !ruby/object:Gem::Requirement
179
+ none: false
158
180
  requirements:
159
- - - '>='
181
+ - - ! '>='
160
182
  - !ruby/object:Gem::Version
161
183
  version: '0'
162
184
  type: :development
163
185
  prerelease: false
164
186
  version_requirements: !ruby/object:Gem::Requirement
187
+ none: false
165
188
  requirements:
166
- - - '>='
189
+ - - ! '>='
167
190
  - !ruby/object:Gem::Version
168
191
  version: '0'
169
192
  description: Google Protocol Buffers serialization and RPC implementation for Ruby.
@@ -352,95 +375,33 @@ files:
352
375
  homepage: https://github.com/localshred/protobuf
353
376
  licenses:
354
377
  - WTFPL
355
- metadata: {}
356
378
  post_install_message:
357
379
  rdoc_options: []
358
380
  require_paths:
359
381
  - lib
360
382
  required_ruby_version: !ruby/object:Gem::Requirement
383
+ none: false
361
384
  requirements:
362
- - - '>='
385
+ - - ! '>='
363
386
  - !ruby/object:Gem::Version
364
387
  version: '0'
388
+ segments:
389
+ - 0
390
+ hash: 3657453006222602873
365
391
  required_rubygems_version: !ruby/object:Gem::Requirement
392
+ none: false
366
393
  requirements:
367
- - - '>='
394
+ - - ! '>='
368
395
  - !ruby/object:Gem::Version
369
396
  version: '0'
397
+ segments:
398
+ - 0
399
+ hash: 3657453006222602873
370
400
  requirements: []
371
401
  rubyforge_project:
372
- rubygems_version: 2.1.2
402
+ rubygems_version: 1.8.24
373
403
  signing_key:
374
- specification_version: 4
404
+ specification_version: 3
375
405
  summary: Google Protocol Buffers serialization and RPC implementation for Ruby.
376
- test_files:
377
- - spec/benchmark/tasks.rb
378
- - spec/bin/protoc-gen-ruby_spec.rb
379
- - spec/data/data.bin
380
- - spec/data/types.bin
381
- - spec/encoding/all_types_spec.rb
382
- - spec/encoding/extreme_values_spec.rb
383
- - spec/functional/embedded_service_spec.rb
384
- - spec/functional/evented_server_spec.rb
385
- - spec/functional/socket_server_spec.rb
386
- - spec/functional/zmq_server_spec.rb
387
- - spec/lib/protobuf/cli_spec.rb
388
- - spec/lib/protobuf/code_generator_spec.rb
389
- - spec/lib/protobuf/enum_spec.rb
390
- - spec/lib/protobuf/enum_value_spec.rb
391
- - spec/lib/protobuf/field/int32_field_spec.rb
392
- - spec/lib/protobuf/field/string_field_spec.rb
393
- - spec/lib/protobuf/generators/base_spec.rb
394
- - spec/lib/protobuf/generators/enum_generator_spec.rb
395
- - spec/lib/protobuf/generators/extension_generator_spec.rb
396
- - spec/lib/protobuf/generators/field_generator_spec.rb
397
- - spec/lib/protobuf/generators/file_generator_spec.rb
398
- - spec/lib/protobuf/generators/message_generator_spec.rb
399
- - spec/lib/protobuf/generators/service_generator_spec.rb
400
- - spec/lib/protobuf/lifecycle_spec.rb
401
- - spec/lib/protobuf/logger_spec.rb
402
- - spec/lib/protobuf/message_spec.rb
403
- - spec/lib/protobuf/rpc/client_spec.rb
404
- - spec/lib/protobuf/rpc/connector_spec.rb
405
- - spec/lib/protobuf/rpc/connectors/base_spec.rb
406
- - spec/lib/protobuf/rpc/connectors/common_spec.rb
407
- - spec/lib/protobuf/rpc/connectors/socket_spec.rb
408
- - spec/lib/protobuf/rpc/connectors/zmq_spec.rb
409
- - spec/lib/protobuf/rpc/servers/evented_server_spec.rb
410
- - spec/lib/protobuf/rpc/servers/socket_server_spec.rb
411
- - spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
412
- - spec/lib/protobuf/rpc/servers/zmq/util_spec.rb
413
- - spec/lib/protobuf/rpc/servers/zmq/worker_spec.rb
414
- - spec/lib/protobuf/rpc/service_directory_spec.rb
415
- - spec/lib/protobuf/rpc/service_dispatcher_spec.rb
416
- - spec/lib/protobuf/rpc/service_filters_spec.rb
417
- - spec/lib/protobuf/rpc/service_spec.rb
418
- - spec/lib/protobuf/rpc/stat_spec.rb
419
- - spec/lib/protobuf_spec.rb
420
- - spec/spec_helper.rb
421
- - spec/support/all.rb
422
- - spec/support/packed_field.rb
423
- - spec/support/server.rb
424
- - spec/support/test/all_types.data.bin
425
- - spec/support/test/all_types.data.txt
426
- - spec/support/test/defaults.pb.rb
427
- - spec/support/test/defaults.proto
428
- - spec/support/test/enum.pb.rb
429
- - spec/support/test/enum.proto
430
- - spec/support/test/extended.pb.rb
431
- - spec/support/test/extended.proto
432
- - spec/support/test/extreme_values.data.bin
433
- - spec/support/test/google_unittest.pb.rb
434
- - spec/support/test/google_unittest.proto
435
- - spec/support/test/google_unittest_import.pb.rb
436
- - spec/support/test/google_unittest_import.proto
437
- - spec/support/test/google_unittest_import_public.pb.rb
438
- - spec/support/test/google_unittest_import_public.proto
439
- - spec/support/test/multi_field_extensions.pb.rb
440
- - spec/support/test/multi_field_extensions.proto
441
- - spec/support/test/resource.pb.rb
442
- - spec/support/test/resource.proto
443
- - spec/support/test/resource_service.rb
444
- - spec/support/test_app_file.rb
445
- - spec/support/tolerance_matcher.rb
406
+ test_files: []
446
407
  has_rdoc:
checksums.yaml DELETED
@@ -1,7 +0,0 @@
1
- ---
2
- SHA1:
3
- metadata.gz: d18d9e64c2e94ed3544b8ba4bd77df2a6ceb2682
4
- data.tar.gz: 762ce63da936d32f99a2433d05968bb40854b6ba
5
- SHA512:
6
- metadata.gz: 0c7721313d80caba216b386da10a7314a506d90e67d8c0b8f5048d2fd5b662d3dc5804ebf634412d928906bc9f94a8adbe4c2cecd25a9b7b4b5e52421d216eb7
7
- data.tar.gz: 5007276b66611190e65a3a7f434d113ed6dadca7cec003f2003563a9f60e724d274aa633c0cda0e2bfa91c96fdf0c52ba102679334f02128677ccf610bab2289