protobuffy 3.6.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (209) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.rubocop.yml +67 -0
  4. data/.rubocop_todo.yml +145 -0
  5. data/.travis.yml +25 -5
  6. data/CHANGES.md +55 -0
  7. data/CONTRIBUTING.md +1 -1
  8. data/LICENSE.txt +17 -9
  9. data/README.md +13 -12
  10. data/Rakefile +15 -11
  11. data/bin/protoc-gen-ruby +8 -3
  12. data/bin/rpc_server +1 -0
  13. data/examples/lib/example/reverse-client.rb +2 -2
  14. data/install-protobuf.sh +28 -0
  15. data/lib/protobuf.rb +57 -53
  16. data/lib/protobuf/cli.rb +94 -74
  17. data/lib/protobuf/code_generator.rb +60 -9
  18. data/lib/protobuf/decoder.rb +19 -65
  19. data/lib/protobuf/deprecation.rb +117 -0
  20. data/lib/protobuf/descriptors/google/protobuf/compiler/plugin.pb.rb +11 -1
  21. data/lib/protobuf/descriptors/google/protobuf/descriptor.pb.rb +55 -3
  22. data/lib/protobuf/encoder.rb +13 -53
  23. data/lib/protobuf/enum.rb +58 -63
  24. data/lib/protobuf/field.rb +4 -4
  25. data/lib/protobuf/field/base_field.rb +101 -173
  26. data/lib/protobuf/field/bool_field.rb +17 -11
  27. data/lib/protobuf/field/bytes_field.rb +21 -35
  28. data/lib/protobuf/field/double_field.rb +0 -1
  29. data/lib/protobuf/field/enum_field.rb +23 -22
  30. data/lib/protobuf/field/field_array.rb +5 -4
  31. data/lib/protobuf/field/fixed32_field.rb +1 -1
  32. data/lib/protobuf/field/fixed64_field.rb +0 -1
  33. data/lib/protobuf/field/float_field.rb +4 -1
  34. data/lib/protobuf/field/int32_field.rb +0 -1
  35. data/lib/protobuf/field/int64_field.rb +0 -1
  36. data/lib/protobuf/field/integer_field.rb +0 -1
  37. data/lib/protobuf/field/message_field.rb +13 -28
  38. data/lib/protobuf/field/sfixed32_field.rb +0 -1
  39. data/lib/protobuf/field/sfixed64_field.rb +0 -1
  40. data/lib/protobuf/field/signed_integer_field.rb +0 -1
  41. data/lib/protobuf/field/sint32_field.rb +0 -1
  42. data/lib/protobuf/field/sint64_field.rb +0 -1
  43. data/lib/protobuf/field/string_field.rb +2 -4
  44. data/lib/protobuf/field/uint32_field.rb +0 -1
  45. data/lib/protobuf/field/uint64_field.rb +0 -1
  46. data/lib/protobuf/field/varint_field.rb +30 -13
  47. data/lib/protobuf/generators/base.rb +30 -16
  48. data/lib/protobuf/generators/enum_generator.rb +6 -9
  49. data/lib/protobuf/generators/extension_generator.rb +1 -2
  50. data/lib/protobuf/generators/field_generator.rb +25 -13
  51. data/lib/protobuf/generators/file_generator.rb +157 -35
  52. data/lib/protobuf/generators/group_generator.rb +22 -17
  53. data/lib/protobuf/generators/message_generator.rb +13 -14
  54. data/lib/protobuf/generators/option_generator.rb +17 -0
  55. data/lib/protobuf/generators/printable.rb +12 -13
  56. data/lib/protobuf/generators/service_generator.rb +2 -3
  57. data/lib/protobuf/http.rb +2 -2
  58. data/lib/protobuf/lifecycle.rb +20 -33
  59. data/lib/protobuf/logging.rb +39 -0
  60. data/lib/protobuf/message.rb +114 -47
  61. data/lib/protobuf/message/fields.rb +170 -88
  62. data/lib/protobuf/message/serialization.rb +19 -18
  63. data/lib/protobuf/optionable.rb +53 -6
  64. data/lib/protobuf/rpc/buffer.rb +18 -19
  65. data/lib/protobuf/rpc/client.rb +22 -50
  66. data/lib/protobuf/rpc/connectors/base.rb +177 -12
  67. data/lib/protobuf/rpc/connectors/http.rb +14 -9
  68. data/lib/protobuf/rpc/connectors/ping.rb +89 -0
  69. data/lib/protobuf/rpc/connectors/socket.rb +13 -8
  70. data/lib/protobuf/rpc/connectors/zmq.rb +178 -73
  71. data/lib/protobuf/rpc/dynamic_discovery.pb.rb +4 -1
  72. data/lib/protobuf/rpc/env.rb +12 -12
  73. data/lib/protobuf/rpc/error.rb +3 -3
  74. data/lib/protobuf/rpc/error/client_error.rb +4 -4
  75. data/lib/protobuf/rpc/error/server_error.rb +9 -9
  76. data/lib/protobuf/rpc/middleware/exception_handler.rb +6 -2
  77. data/lib/protobuf/rpc/middleware/logger.rb +8 -4
  78. data/lib/protobuf/rpc/middleware/request_decoder.rb +17 -21
  79. data/lib/protobuf/rpc/middleware/response_encoder.rb +22 -27
  80. data/lib/protobuf/rpc/middleware/statsd.rb +3 -3
  81. data/lib/protobuf/rpc/rpc.pb.rb +4 -1
  82. data/lib/protobuf/rpc/server.rb +1 -1
  83. data/lib/protobuf/rpc/servers/http/server.rb +19 -17
  84. data/lib/protobuf/rpc/servers/socket/server.rb +78 -70
  85. data/lib/protobuf/rpc/servers/socket/worker.rb +4 -4
  86. data/lib/protobuf/rpc/servers/socket_runner.rb +27 -15
  87. data/lib/protobuf/rpc/servers/zmq/broker.rb +70 -31
  88. data/lib/protobuf/rpc/servers/zmq/server.rb +55 -47
  89. data/lib/protobuf/rpc/servers/zmq/util.rb +14 -13
  90. data/lib/protobuf/rpc/servers/zmq/worker.rb +16 -16
  91. data/lib/protobuf/rpc/servers/zmq_runner.rb +26 -7
  92. data/lib/protobuf/rpc/service.rb +21 -27
  93. data/lib/protobuf/rpc/service_directory.rb +43 -27
  94. data/lib/protobuf/rpc/service_dispatcher.rb +9 -10
  95. data/lib/protobuf/rpc/service_filters.rb +32 -55
  96. data/lib/protobuf/rpc/stat.rb +4 -8
  97. data/lib/protobuf/socket.rb +1 -2
  98. data/lib/protobuf/tasks/compile.rake +3 -4
  99. data/lib/protobuf/varint.rb +9 -0
  100. data/lib/protobuf/varint_pure.rb +13 -0
  101. data/lib/protobuf/version.rb +1 -1
  102. data/lib/protobuf/zmq.rb +2 -2
  103. data/proto/google/protobuf/descriptor.proto +190 -31
  104. data/protobuffy.gemspec +30 -17
  105. data/spec/benchmark/tasks.rb +27 -19
  106. data/spec/bin/protoc-gen-ruby_spec.rb +11 -6
  107. data/spec/encoding/all_types_spec.rb +96 -84
  108. data/spec/encoding/extreme_values_spec.rb +0 -0
  109. data/spec/functional/class_inheritance_spec.rb +52 -0
  110. data/spec/functional/code_generator_spec.rb +38 -0
  111. data/spec/functional/socket_server_spec.rb +15 -15
  112. data/spec/functional/zmq_server_spec.rb +29 -27
  113. data/spec/lib/protobuf/cli_spec.rb +82 -67
  114. data/spec/lib/protobuf/code_generator_spec.rb +37 -10
  115. data/spec/lib/protobuf/enum_spec.rb +77 -46
  116. data/spec/lib/protobuf/field/bool_field_spec.rb +91 -0
  117. data/spec/lib/protobuf/field/double_field_spec.rb +9 -0
  118. data/spec/lib/protobuf/field/enum_field_spec.rb +26 -0
  119. data/spec/lib/protobuf/field/field_array_spec.rb +69 -0
  120. data/spec/lib/protobuf/field/fixed32_field_spec.rb +7 -0
  121. data/spec/lib/protobuf/field/fixed64_field_spec.rb +7 -0
  122. data/spec/lib/protobuf/field/float_field_spec.rb +90 -0
  123. data/spec/lib/protobuf/field/int32_field_spec.rb +114 -1
  124. data/spec/lib/protobuf/field/int64_field_spec.rb +7 -0
  125. data/spec/lib/protobuf/field/message_field_spec.rb +132 -0
  126. data/spec/lib/protobuf/field/sfixed32_field_spec.rb +9 -0
  127. data/spec/lib/protobuf/field/sfixed64_field_spec.rb +9 -0
  128. data/spec/lib/protobuf/field/sint32_field_spec.rb +9 -0
  129. data/spec/lib/protobuf/field/sint64_field_spec.rb +9 -0
  130. data/spec/lib/protobuf/field/string_field_spec.rb +44 -11
  131. data/spec/lib/protobuf/field/uint32_field_spec.rb +7 -0
  132. data/spec/lib/protobuf/field/uint64_field_spec.rb +7 -0
  133. data/spec/lib/protobuf/field_spec.rb +4 -6
  134. data/spec/lib/protobuf/generators/base_spec.rb +80 -13
  135. data/spec/lib/protobuf/generators/enum_generator_spec.rb +35 -21
  136. data/spec/lib/protobuf/generators/extension_generator_spec.rb +12 -13
  137. data/spec/lib/protobuf/generators/field_generator_spec.rb +73 -21
  138. data/spec/lib/protobuf/generators/file_generator_spec.rb +89 -6
  139. data/spec/lib/protobuf/generators/service_generator_spec.rb +25 -13
  140. data/spec/lib/protobuf/lifecycle_spec.rb +25 -20
  141. data/spec/lib/protobuf/message_spec.rb +578 -79
  142. data/spec/lib/protobuf/optionable_spec.rb +202 -26
  143. data/spec/lib/protobuf/rpc/client_spec.rb +16 -16
  144. data/spec/lib/protobuf/rpc/connectors/base_spec.rb +167 -13
  145. data/spec/lib/protobuf/rpc/connectors/connector_spec.rb +4 -5
  146. data/spec/lib/protobuf/rpc/connectors/http_spec.rb +13 -11
  147. data/spec/lib/protobuf/rpc/connectors/ping_spec.rb +69 -0
  148. data/spec/lib/protobuf/rpc/connectors/socket_spec.rb +6 -7
  149. data/spec/lib/protobuf/rpc/connectors/zmq_spec.rb +35 -52
  150. data/spec/lib/protobuf/rpc/middleware/exception_handler_spec.rb +10 -10
  151. data/spec/lib/protobuf/rpc/middleware/logger_spec.rb +11 -11
  152. data/spec/lib/protobuf/rpc/middleware/request_decoder_spec.rb +23 -23
  153. data/spec/lib/protobuf/rpc/middleware/response_encoder_spec.rb +11 -11
  154. data/spec/lib/protobuf/rpc/middleware/statsd_spec.rb +6 -6
  155. data/spec/lib/protobuf/rpc/servers/http/server_spec.rb +47 -44
  156. data/spec/lib/protobuf/rpc/servers/socket_server_spec.rb +6 -6
  157. data/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb +12 -10
  158. data/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb +11 -11
  159. data/spec/lib/protobuf/rpc/servers/zmq/worker_spec.rb +7 -7
  160. data/spec/lib/protobuf/rpc/service_directory_spec.rb +47 -49
  161. data/spec/lib/protobuf/rpc/service_dispatcher_spec.rb +8 -25
  162. data/spec/lib/protobuf/rpc/service_filters_spec.rb +102 -69
  163. data/spec/lib/protobuf/rpc/service_spec.rb +37 -36
  164. data/spec/lib/protobuf/rpc/stat_spec.rb +7 -9
  165. data/spec/lib/protobuf/varint_spec.rb +29 -0
  166. data/spec/lib/protobuf_spec.rb +55 -28
  167. data/spec/spec_helper.rb +12 -27
  168. data/spec/support/all.rb +0 -1
  169. data/spec/support/packed_field.rb +4 -3
  170. data/spec/support/{test → protos}/all_types.data.bin +0 -0
  171. data/spec/support/{test → protos}/all_types.data.txt +0 -0
  172. data/spec/support/{test → protos}/enum.pb.rb +8 -4
  173. data/spec/support/{test → protos}/enum.proto +4 -1
  174. data/spec/support/{test → protos}/extreme_values.data.bin +0 -0
  175. data/spec/support/protos/google_unittest.bin +0 -0
  176. data/spec/support/protos/google_unittest.pb.rb +798 -0
  177. data/spec/support/{test → protos}/google_unittest.proto +237 -66
  178. data/spec/support/protos/google_unittest_custom_options.bin +0 -0
  179. data/spec/support/protos/google_unittest_custom_options.pb.rb +268 -0
  180. data/spec/support/protos/google_unittest_custom_options.proto +424 -0
  181. data/spec/support/protos/google_unittest_import.pb.rb +55 -0
  182. data/spec/support/{test → protos}/google_unittest_import.proto +19 -10
  183. data/spec/support/protos/google_unittest_import_public.pb.rb +31 -0
  184. data/spec/support/{test → protos}/google_unittest_import_public.proto +8 -5
  185. data/spec/support/{test → protos}/multi_field_extensions.pb.rb +5 -2
  186. data/spec/support/{test → protos}/multi_field_extensions.proto +2 -0
  187. data/spec/support/{test → protos}/resource.pb.rb +47 -11
  188. data/spec/support/{test → protos}/resource.proto +24 -1
  189. data/spec/support/resource_service.rb +23 -0
  190. data/spec/support/server.rb +32 -61
  191. metadata +119 -59
  192. data/lib/protobuf/deprecator.rb +0 -42
  193. data/lib/protobuf/logger.rb +0 -93
  194. data/lib/protobuf/rpc/connector.rb +0 -21
  195. data/lib/protobuf/rpc/connectors/common.rb +0 -172
  196. data/spec/data/data.bin +0 -3
  197. data/spec/data/types.bin +0 -0
  198. data/spec/lib/protobuf/logger_spec.rb +0 -145
  199. data/spec/lib/protobuf/rpc/connector_spec.rb +0 -26
  200. data/spec/lib/protobuf/rpc/connectors/common_spec.rb +0 -170
  201. data/spec/support/test/defaults.pb.rb +0 -25
  202. data/spec/support/test/defaults.proto +0 -9
  203. data/spec/support/test/extended.pb.rb +0 -22
  204. data/spec/support/test/extended.proto +0 -10
  205. data/spec/support/test/google_unittest.pb.rb +0 -543
  206. data/spec/support/test/google_unittest_import.pb.rb +0 -37
  207. data/spec/support/test/google_unittest_import_public.pb.rb +0 -8
  208. data/spec/support/test/resource_service.rb +0 -26
  209. data/spec/support/tolerance_matcher.rb +0 -40
@@ -1,111 +1,119 @@
1
- require 'protobuf/rpc/server'
1
+ require 'set'
2
+
2
3
  require 'protobuf/rpc/servers/socket/worker'
3
4
 
4
5
  module Protobuf
5
6
  module Rpc
6
7
  module Socket
7
8
  class Server
8
- include ::Protobuf::Logger::LogMethods
9
+ include ::Protobuf::Logging
9
10
 
10
11
  AUTO_COLLECT_TIMEOUT = 5 # seconds
11
12
 
13
+ private
14
+
15
+ attr_accessor :threshold, :host, :port, :backlog
16
+ attr_writer :running
17
+
18
+ public
19
+
20
+ attr_reader :running
21
+ alias :running? running
22
+
12
23
  def initialize(options)
13
- @options = options
24
+ self.running = false
25
+ self.host = options.fetch(:host)
26
+ self.port = options.fetch(:port)
27
+ self.backlog = options.fetch(:backlog, 100)
28
+ self.threshold = options.fetch(:threshold, 100)
29
+ end
30
+
31
+ def threads
32
+ @threads ||= []
33
+ end
34
+
35
+ def working
36
+ @working ||= Set.new
14
37
  end
15
38
 
16
39
  def cleanup?
17
- # every 10 connections run a cleanup routine after closing the response
18
- @threads.size > (@threshold - 1) && (@threads.size % @threshold) == 0
40
+ # every `threshold` connections run a cleanup routine after closing the response
41
+ !threads.empty? && threads.size % threshold == 0
19
42
  end
20
43
 
21
44
  def cleanup_threads
22
- log_debug { sign_message("Thread cleanup - #{@threads.size} - start") }
23
-
24
- @threads = @threads.select do |t|
25
- if t[:thread].alive?
26
- true
27
- else
28
- t[:thread].join
29
- @working.delete(t[:socket])
30
- false
45
+ logger.debug { sign_message("Thread cleanup - #{threads.size} - start") }
46
+
47
+ threads.delete_if do |hash|
48
+ unless (thread = hash.fetch(:thread)).alive?
49
+ thread.join
50
+ working.delete(hash.fetch(:socket))
31
51
  end
32
52
  end
33
53
 
34
- log_debug { sign_message("Thread cleanup - #{@threads.size} - complete") }
54
+ logger.debug { sign_message("Thread cleanup - #{threads.size} - complete") }
35
55
  end
36
56
 
37
57
  def log_signature
38
- @_log_signature ||= "server-#{self.class.name}"
58
+ @_log_signature ||= "[server-#{self.class.name}]"
39
59
  end
40
60
 
41
61
  def new_worker(socket)
42
62
  Thread.new(socket) do |sock|
43
- ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
44
- s.close
45
- end
63
+ ::Protobuf::Rpc::Socket::Worker.new(sock, &:close)
46
64
  end
47
65
  end
48
66
 
49
67
  def run
50
- log_debug { sign_message("Run") }
51
- host = @options[:host]
52
- port = @options[:port]
53
- backlog = @options[:backlog]
54
- @threshold = @options[:threshold]
55
-
56
- @threads = []
57
- @server = ::TCPServer.new(host, port)
58
- raise "The server was unable to start properly." if @server.closed?
59
-
60
- @server.listen(backlog)
61
- @working = []
62
- @listen_fds = [@server]
63
- @running = true
64
-
65
- while running?
66
- log_debug { sign_message("Waiting for connections") }
67
- ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT) rescue nil
68
-
69
- if ready_cnxns
70
- cnxns = ready_cnxns.first
71
- cnxns.each do |client|
72
- case
73
- when !running? then
74
- # no-op
75
- when client == @server then
76
- log_debug { sign_message("Accepted new connection") }
77
- client, sockaddr = @server.accept
78
- @listen_fds << client
79
- else
80
- unless @working.include?(client)
81
- @working << @listen_fds.delete(client)
82
- log_debug { sign_message("Working") }
83
- @threads << { :thread => new_worker(client), :socket => client }
84
-
85
- cleanup_threads if cleanup?
68
+ logger.debug { sign_message("Run") }
69
+
70
+ server = ::TCPServer.new(host, port)
71
+ fail "The server was unable to start properly." if server.closed?
72
+
73
+ begin
74
+ server.listen(backlog)
75
+ listen_fds = [server]
76
+ self.running = true
77
+
78
+ while running?
79
+ logger.debug { sign_message("Waiting for connections") }
80
+ ready_cnxns = begin
81
+ IO.select(listen_fds, [], [], AUTO_COLLECT_TIMEOUT)
82
+ rescue IOError
83
+ nil
84
+ end
85
+
86
+ if ready_cnxns
87
+ ready_cnxns.first.each do |client|
88
+ case
89
+ when !running?
90
+ # no-op
91
+ when client == server
92
+ logger.debug { sign_message("Accepted new connection") }
93
+ client, _sockaddr = server.accept
94
+ listen_fds << client
95
+ else
96
+ unless working.include?(client)
97
+ working << listen_fds.delete(client)
98
+ logger.debug { sign_message("Working") }
99
+ threads << { :thread => new_worker(client), :socket => client }
100
+
101
+ cleanup_threads if cleanup?
102
+ end
86
103
  end
87
104
  end
105
+ elsif threads.size > 1
106
+ # Run a cleanup if select times out while waiting
107
+ cleanup_threads
88
108
  end
89
- else
90
- # Run a cleanup if select times out while waiting
91
- cleanup_threads if @threads.size > 1
92
109
  end
110
+ ensure
111
+ server.close
93
112
  end
94
-
95
- rescue Errno::EADDRINUSE
96
- raise
97
- rescue
98
- # Closing the server causes the loop to raise an exception here
99
- raise #if running?
100
- end
101
-
102
- def running?
103
- !!@running
104
113
  end
105
114
 
106
115
  def stop
107
- @running = false
108
- @server.try(:close)
116
+ self.running = false
109
117
  end
110
118
  end
111
119
  end
@@ -1,12 +1,12 @@
1
1
  require 'protobuf/rpc/server'
2
- require 'protobuf/logger'
2
+ require 'protobuf/logging'
3
3
 
4
4
  module Protobuf
5
5
  module Rpc
6
6
  module Socket
7
7
  class Worker
8
8
  include ::Protobuf::Rpc::Server
9
- include ::Protobuf::Logger::LogMethods
9
+ include ::Protobuf::Logging
10
10
 
11
11
  def initialize(sock, &complete_cb)
12
12
  @socket = sock
@@ -33,7 +33,7 @@ module Protobuf
33
33
  end
34
34
 
35
35
  def send_data(data)
36
- raise 'Socket closed unexpectedly' unless socket_writable?
36
+ fail 'Socket closed unexpectedly' unless socket_writable?
37
37
  response_buffer = Protobuf::Rpc::Buffer.new(:write)
38
38
  response_buffer.set_data(data)
39
39
 
@@ -44,7 +44,7 @@ module Protobuf
44
44
  end
45
45
 
46
46
  def log_signature
47
- @_log_signature ||= "server-#{self.class}-#{object_id}"
47
+ @_log_signature ||= "[server-#{self.class}-#{object_id}]"
48
48
  end
49
49
 
50
50
  def socket_writable?
@@ -2,33 +2,45 @@ module Protobuf
2
2
  module Rpc
3
3
  class SocketRunner
4
4
 
5
+ private
6
+
7
+ attr_accessor :server
8
+
9
+ public
10
+
5
11
  def initialize(options)
6
- @options = case
7
- when options.is_a?(OpenStruct) then
8
- options.marshal_dump
9
- when options.is_a?(Hash) then
10
- options
11
- when options.respond_to?(:to_hash) then
12
- options.to_hash
13
- else
14
- raise "Cannot parser Socket Server - server options"
15
- end
16
-
17
- @server = ::Protobuf::Rpc::Socket::Server.new(@options)
12
+ options = case
13
+ when options.is_a?(OpenStruct) then
14
+ options.marshal_dump
15
+ when options.respond_to?(:to_hash) then
16
+ options.to_hash.symbolize_keys
17
+ else
18
+ fail "Cannot parser Socket Server - server options"
19
+ end
20
+
21
+ self.server = ::Protobuf::Rpc::Socket::Server.new(options)
18
22
  end
19
23
 
20
24
  def run
21
25
  yield if block_given?
22
- @server.run
26
+ server.run
23
27
  end
24
28
 
25
29
  def running?
26
- @server.running?
30
+ server.running?
27
31
  end
28
32
 
29
33
  def stop
30
- @server.stop
34
+ server.stop
31
35
  end
32
36
  end
33
37
  end
34
38
  end
39
+
40
+ module Protobuf
41
+ module Rpc
42
+ module Servers # bad file namespacing
43
+ SocketRunner = ::Protobuf::Rpc::SocketRunner
44
+ end
45
+ end
46
+ end
@@ -23,39 +23,69 @@ module Protobuf
23
23
 
24
24
  def run
25
25
  @idle_workers = []
26
+ @running = true
26
27
 
27
28
  loop do
28
- unless local_queue.empty?
29
- process_local_queue
30
- end
31
-
32
- rc = @poller.poll(500)
29
+ process_local_queue
30
+ rc = @poller.poll(broker_polling_milliseconds)
33
31
 
34
32
  # The server was shutdown and no requests are pending
35
- break if rc == 0 && !running?
36
-
33
+ break if rc == 0 && !running? && @server.workers.empty?
37
34
  # Something went wrong
38
35
  break if rc == -1
39
36
 
40
- @poller.readables.each do |readable|
41
- case readable
42
- when @frontend_socket
43
- process_frontend
44
- when @backend_socket
45
- process_backend
46
- end
47
- end
37
+ check_and_process_backend
38
+ process_local_queue # Fair ordering so queued requests get in before new requests
39
+ check_and_process_frontend
48
40
  end
49
41
  ensure
50
42
  teardown
43
+ @running = false
51
44
  end
52
45
 
53
46
  def running?
54
- @server.running? || @server.workers.any?
47
+ @running && @server.running?
55
48
  end
56
49
 
57
50
  private
58
51
 
52
+ def backend_poll_weight
53
+ @backend_poll_weight ||= [ENV["PB_ZMQ_SERVER_BACKEND_POLL_WEIGHT"].to_i, 1].max
54
+ end
55
+
56
+ def broker_polling_milliseconds
57
+ @broker_polling_milliseconds ||= [ENV["PB_ZMQ_BROKER_POLLING_MILLISECONDS"].to_i, 500].max
58
+ end
59
+
60
+ def check_and_process_backend
61
+ readables_include_backend = @poller.readables.include?(@backend_socket)
62
+ message_count_read_from_backend = 0
63
+
64
+ while readables_include_backend && message_count_read_from_backend < backend_poll_weight
65
+ message_count_read_from_backend += 1
66
+ process_backend
67
+ @poller.poll_nonblock
68
+ readables_include_backend = @poller.readables.include?(@backend_socket)
69
+ end
70
+ end
71
+
72
+ def check_and_process_frontend
73
+ readables_include_frontend = @poller.readables.include?(@frontend_socket)
74
+ message_count_read_from_frontend = 0
75
+
76
+ while readables_include_frontend && message_count_read_from_frontend < frontend_poll_weight
77
+ message_count_read_from_frontend += 1
78
+ process_frontend
79
+ break unless local_queue_available? # no need to read frontend just to throw away messages, will prioritize backend when full
80
+ @poller.poll_nonblock
81
+ readables_include_frontend = @poller.readables.include?(@frontend_socket)
82
+ end
83
+ end
84
+
85
+ def frontend_poll_weight
86
+ @frontend_poll_weight ||= [ENV["PB_ZMQ_SERVER_FRONTEND_POLL_WEIGHT"].to_i, 1].max
87
+ end
88
+
59
89
  def init_backend_socket
60
90
  @backend_socket = @zmq_context.socket(ZMQ::ROUTER)
61
91
  zmq_error_check(@backend_socket.bind(@server.backend_uri))
@@ -67,7 +97,7 @@ module Protobuf
67
97
  end
68
98
 
69
99
  def init_local_queue
70
- @local_queue = ::Queue.new
100
+ @local_queue = []
71
101
  end
72
102
 
73
103
  def init_poller
@@ -77,19 +107,28 @@ module Protobuf
77
107
  end
78
108
 
79
109
  def init_zmq_context
80
- if inproc?
81
- @zmq_context = @server.zmq_context
82
- else
83
- @zmq_context = ZMQ::Context.new
84
- end
110
+ @zmq_context =
111
+ if inproc?
112
+ @server.zmq_context
113
+ else
114
+ ZMQ::Context.new
115
+ end
85
116
  end
86
117
 
87
118
  def inproc?
88
119
  !!@server.try(:inproc?)
89
120
  end
90
121
 
122
+ def local_queue_available?
123
+ local_queue.size < local_queue_max_size && running?
124
+ end
125
+
126
+ def local_queue_max_size
127
+ @local_queue_max_size ||= [ENV["PB_ZMQ_SERVER_QUEUE_MAX_SIZE"].to_i, 5].max
128
+ end
129
+
91
130
  def process_backend
92
- worker, ignore, *frames = read_from_backend
131
+ worker, _ignore, *frames = read_from_backend
93
132
 
94
133
  @idle_workers << worker
95
134
 
@@ -102,16 +141,16 @@ module Protobuf
102
141
  address, _, message, *frames = read_from_frontend
103
142
 
104
143
  if message == ::Protobuf::Rpc::Zmq::CHECK_AVAILABLE_MESSAGE
105
- if @idle_workers.any? || local_queue.empty?
106
- write_to_frontend([address, "", ::Protobuf::Rpc::Zmq::WORKERS_AVAILABLE])
144
+ if local_queue_available?
145
+ write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::WORKERS_AVAILABLE])
107
146
  else
108
- write_to_frontend([address, "", ::Protobuf::Rpc::Zmq::NO_WORKERS_AVAILABLE])
147
+ write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::NO_WORKERS_AVAILABLE])
109
148
  end
110
149
  else
111
- if @idle_workers.any?
112
- write_to_backend([@idle_workers.shift, ""] + [address, "", message ] + frames)
150
+ if @idle_workers.empty? # rubocop:disable Style/IfInsideElse
151
+ local_queue << [address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message].concat(frames)
113
152
  else
114
- local_queue.push([address, "", message ] + frames)
153
+ write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message]).concat(frames))
115
154
  end
116
155
  end
117
156
  end
@@ -120,7 +159,7 @@ module Protobuf
120
159
  return if local_queue.empty?
121
160
  return if @idle_workers.empty?
122
161
 
123
- write_to_backend([@idle_workers.shift, ""] + local_queue.pop)
162
+ write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat(local_queue.shift))
124
163
  process_local_queue
125
164
  end
126
165
 
@@ -139,7 +178,7 @@ module Protobuf
139
178
  def teardown
140
179
  @frontend_socket.try(:close)
141
180
  @backend_socket.try(:close)
142
- @zmq_context.try(:terminate)
181
+ @zmq_context.try(:terminate) unless inproc?
143
182
  end
144
183
 
145
184
  def write_to_backend(frames)