protobuf-cucumber 3.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +21 -0
  3. data/.rubocop.yml +70 -0
  4. data/.rubocop_todo.yml +145 -0
  5. data/.travis.yml +40 -0
  6. data/.yardopts +5 -0
  7. data/CHANGES.md +344 -0
  8. data/CONTRIBUTING.md +16 -0
  9. data/Gemfile +3 -0
  10. data/LICENSE.txt +22 -0
  11. data/README.md +33 -0
  12. data/Rakefile +64 -0
  13. data/bin/protoc-gen-ruby +22 -0
  14. data/bin/rpc_server +5 -0
  15. data/install-protobuf.sh +28 -0
  16. data/lib/protobuf.rb +129 -0
  17. data/lib/protobuf/cli.rb +257 -0
  18. data/lib/protobuf/code_generator.rb +120 -0
  19. data/lib/protobuf/decoder.rb +28 -0
  20. data/lib/protobuf/deprecation.rb +117 -0
  21. data/lib/protobuf/descriptors.rb +3 -0
  22. data/lib/protobuf/descriptors/google/protobuf/compiler/plugin.pb.rb +62 -0
  23. data/lib/protobuf/descriptors/google/protobuf/descriptor.pb.rb +301 -0
  24. data/lib/protobuf/encoder.rb +11 -0
  25. data/lib/protobuf/enum.rb +365 -0
  26. data/lib/protobuf/exceptions.rb +9 -0
  27. data/lib/protobuf/field.rb +74 -0
  28. data/lib/protobuf/field/base_field.rb +380 -0
  29. data/lib/protobuf/field/base_field_object_definitions.rb +504 -0
  30. data/lib/protobuf/field/bool_field.rb +64 -0
  31. data/lib/protobuf/field/bytes_field.rb +78 -0
  32. data/lib/protobuf/field/double_field.rb +25 -0
  33. data/lib/protobuf/field/enum_field.rb +61 -0
  34. data/lib/protobuf/field/field_array.rb +104 -0
  35. data/lib/protobuf/field/field_hash.rb +122 -0
  36. data/lib/protobuf/field/fixed32_field.rb +25 -0
  37. data/lib/protobuf/field/fixed64_field.rb +28 -0
  38. data/lib/protobuf/field/float_field.rb +43 -0
  39. data/lib/protobuf/field/int32_field.rb +21 -0
  40. data/lib/protobuf/field/int64_field.rb +34 -0
  41. data/lib/protobuf/field/integer_field.rb +23 -0
  42. data/lib/protobuf/field/message_field.rb +51 -0
  43. data/lib/protobuf/field/sfixed32_field.rb +27 -0
  44. data/lib/protobuf/field/sfixed64_field.rb +28 -0
  45. data/lib/protobuf/field/signed_integer_field.rb +29 -0
  46. data/lib/protobuf/field/sint32_field.rb +21 -0
  47. data/lib/protobuf/field/sint64_field.rb +21 -0
  48. data/lib/protobuf/field/string_field.rb +51 -0
  49. data/lib/protobuf/field/uint32_field.rb +21 -0
  50. data/lib/protobuf/field/uint64_field.rb +21 -0
  51. data/lib/protobuf/field/varint_field.rb +77 -0
  52. data/lib/protobuf/generators/base.rb +85 -0
  53. data/lib/protobuf/generators/enum_generator.rb +39 -0
  54. data/lib/protobuf/generators/extension_generator.rb +27 -0
  55. data/lib/protobuf/generators/field_generator.rb +193 -0
  56. data/lib/protobuf/generators/file_generator.rb +262 -0
  57. data/lib/protobuf/generators/group_generator.rb +122 -0
  58. data/lib/protobuf/generators/message_generator.rb +104 -0
  59. data/lib/protobuf/generators/option_generator.rb +17 -0
  60. data/lib/protobuf/generators/printable.rb +160 -0
  61. data/lib/protobuf/generators/service_generator.rb +50 -0
  62. data/lib/protobuf/lifecycle.rb +33 -0
  63. data/lib/protobuf/logging.rb +39 -0
  64. data/lib/protobuf/message.rb +260 -0
  65. data/lib/protobuf/message/fields.rb +233 -0
  66. data/lib/protobuf/message/serialization.rb +85 -0
  67. data/lib/protobuf/optionable.rb +70 -0
  68. data/lib/protobuf/rpc/buffer.rb +78 -0
  69. data/lib/protobuf/rpc/client.rb +140 -0
  70. data/lib/protobuf/rpc/connectors/base.rb +221 -0
  71. data/lib/protobuf/rpc/connectors/ping.rb +89 -0
  72. data/lib/protobuf/rpc/connectors/socket.rb +78 -0
  73. data/lib/protobuf/rpc/connectors/zmq.rb +319 -0
  74. data/lib/protobuf/rpc/dynamic_discovery.pb.rb +50 -0
  75. data/lib/protobuf/rpc/env.rb +60 -0
  76. data/lib/protobuf/rpc/error.rb +28 -0
  77. data/lib/protobuf/rpc/error/client_error.rb +31 -0
  78. data/lib/protobuf/rpc/error/server_error.rb +43 -0
  79. data/lib/protobuf/rpc/middleware.rb +25 -0
  80. data/lib/protobuf/rpc/middleware/exception_handler.rb +40 -0
  81. data/lib/protobuf/rpc/middleware/logger.rb +95 -0
  82. data/lib/protobuf/rpc/middleware/request_decoder.rb +79 -0
  83. data/lib/protobuf/rpc/middleware/response_encoder.rb +83 -0
  84. data/lib/protobuf/rpc/middleware/runner.rb +18 -0
  85. data/lib/protobuf/rpc/rpc.pb.rb +64 -0
  86. data/lib/protobuf/rpc/rpc_method.rb +16 -0
  87. data/lib/protobuf/rpc/server.rb +39 -0
  88. data/lib/protobuf/rpc/servers/socket/server.rb +121 -0
  89. data/lib/protobuf/rpc/servers/socket/worker.rb +56 -0
  90. data/lib/protobuf/rpc/servers/socket_runner.rb +46 -0
  91. data/lib/protobuf/rpc/servers/zmq/broker.rb +194 -0
  92. data/lib/protobuf/rpc/servers/zmq/server.rb +321 -0
  93. data/lib/protobuf/rpc/servers/zmq/util.rb +48 -0
  94. data/lib/protobuf/rpc/servers/zmq/worker.rb +105 -0
  95. data/lib/protobuf/rpc/servers/zmq_runner.rb +70 -0
  96. data/lib/protobuf/rpc/service.rb +172 -0
  97. data/lib/protobuf/rpc/service_directory.rb +261 -0
  98. data/lib/protobuf/rpc/service_dispatcher.rb +45 -0
  99. data/lib/protobuf/rpc/service_filters.rb +250 -0
  100. data/lib/protobuf/rpc/stat.rb +119 -0
  101. data/lib/protobuf/socket.rb +21 -0
  102. data/lib/protobuf/tasks.rb +1 -0
  103. data/lib/protobuf/tasks/compile.rake +80 -0
  104. data/lib/protobuf/varint.rb +20 -0
  105. data/lib/protobuf/varint_pure.rb +31 -0
  106. data/lib/protobuf/version.rb +3 -0
  107. data/lib/protobuf/wire_type.rb +10 -0
  108. data/lib/protobuf/zmq.rb +21 -0
  109. data/profile.html +5103 -0
  110. data/proto/dynamic_discovery.proto +44 -0
  111. data/proto/google/protobuf/compiler/plugin.proto +147 -0
  112. data/proto/google/protobuf/descriptor.proto +779 -0
  113. data/proto/rpc.proto +69 -0
  114. data/protobuf-cucumber.gemspec +57 -0
  115. data/spec/benchmark/tasks.rb +143 -0
  116. data/spec/bin/protoc-gen-ruby_spec.rb +23 -0
  117. data/spec/encoding/all_types_spec.rb +103 -0
  118. data/spec/encoding/extreme_values_spec.rb +0 -0
  119. data/spec/functional/class_inheritance_spec.rb +52 -0
  120. data/spec/functional/code_generator_spec.rb +58 -0
  121. data/spec/functional/socket_server_spec.rb +59 -0
  122. data/spec/functional/zmq_server_spec.rb +105 -0
  123. data/spec/lib/protobuf/cli_spec.rb +317 -0
  124. data/spec/lib/protobuf/code_generator_spec.rb +87 -0
  125. data/spec/lib/protobuf/enum_spec.rb +307 -0
  126. data/spec/lib/protobuf/field/bool_field_spec.rb +91 -0
  127. data/spec/lib/protobuf/field/double_field_spec.rb +9 -0
  128. data/spec/lib/protobuf/field/enum_field_spec.rb +44 -0
  129. data/spec/lib/protobuf/field/field_array_spec.rb +105 -0
  130. data/spec/lib/protobuf/field/field_hash_spec.rb +168 -0
  131. data/spec/lib/protobuf/field/fixed32_field_spec.rb +7 -0
  132. data/spec/lib/protobuf/field/fixed64_field_spec.rb +7 -0
  133. data/spec/lib/protobuf/field/float_field_spec.rb +90 -0
  134. data/spec/lib/protobuf/field/int32_field_spec.rb +120 -0
  135. data/spec/lib/protobuf/field/int64_field_spec.rb +7 -0
  136. data/spec/lib/protobuf/field/message_field_spec.rb +132 -0
  137. data/spec/lib/protobuf/field/sfixed32_field_spec.rb +9 -0
  138. data/spec/lib/protobuf/field/sfixed64_field_spec.rb +9 -0
  139. data/spec/lib/protobuf/field/sint32_field_spec.rb +9 -0
  140. data/spec/lib/protobuf/field/sint64_field_spec.rb +9 -0
  141. data/spec/lib/protobuf/field/string_field_spec.rb +79 -0
  142. data/spec/lib/protobuf/field/uint32_field_spec.rb +7 -0
  143. data/spec/lib/protobuf/field/uint64_field_spec.rb +7 -0
  144. data/spec/lib/protobuf/field_spec.rb +192 -0
  145. data/spec/lib/protobuf/generators/base_spec.rb +154 -0
  146. data/spec/lib/protobuf/generators/enum_generator_spec.rb +82 -0
  147. data/spec/lib/protobuf/generators/extension_generator_spec.rb +42 -0
  148. data/spec/lib/protobuf/generators/field_generator_spec.rb +197 -0
  149. data/spec/lib/protobuf/generators/file_generator_spec.rb +119 -0
  150. data/spec/lib/protobuf/generators/message_generator_spec.rb +0 -0
  151. data/spec/lib/protobuf/generators/service_generator_spec.rb +99 -0
  152. data/spec/lib/protobuf/lifecycle_spec.rb +94 -0
  153. data/spec/lib/protobuf/message_spec.rb +944 -0
  154. data/spec/lib/protobuf/optionable_spec.rb +265 -0
  155. data/spec/lib/protobuf/rpc/client_spec.rb +66 -0
  156. data/spec/lib/protobuf/rpc/connectors/base_spec.rb +226 -0
  157. data/spec/lib/protobuf/rpc/connectors/ping_spec.rb +69 -0
  158. data/spec/lib/protobuf/rpc/connectors/socket_spec.rb +34 -0
  159. data/spec/lib/protobuf/rpc/connectors/zmq_spec.rb +110 -0
  160. data/spec/lib/protobuf/rpc/middleware/exception_handler_spec.rb +62 -0
  161. data/spec/lib/protobuf/rpc/middleware/logger_spec.rb +49 -0
  162. data/spec/lib/protobuf/rpc/middleware/request_decoder_spec.rb +115 -0
  163. data/spec/lib/protobuf/rpc/middleware/response_encoder_spec.rb +91 -0
  164. data/spec/lib/protobuf/rpc/servers/socket_server_spec.rb +38 -0
  165. data/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb +43 -0
  166. data/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb +55 -0
  167. data/spec/lib/protobuf/rpc/servers/zmq/worker_spec.rb +35 -0
  168. data/spec/lib/protobuf/rpc/service_directory_spec.rb +293 -0
  169. data/spec/lib/protobuf/rpc/service_dispatcher_spec.rb +35 -0
  170. data/spec/lib/protobuf/rpc/service_filters_spec.rb +517 -0
  171. data/spec/lib/protobuf/rpc/service_spec.rb +162 -0
  172. data/spec/lib/protobuf/rpc/stat_spec.rb +101 -0
  173. data/spec/lib/protobuf/varint_spec.rb +29 -0
  174. data/spec/lib/protobuf_spec.rb +105 -0
  175. data/spec/spec_helper.rb +42 -0
  176. data/spec/support/all.rb +6 -0
  177. data/spec/support/packed_field.rb +23 -0
  178. data/spec/support/protos/all_types.data.bin +0 -0
  179. data/spec/support/protos/all_types.data.txt +119 -0
  180. data/spec/support/protos/enum.pb.rb +63 -0
  181. data/spec/support/protos/enum.proto +37 -0
  182. data/spec/support/protos/extreme_values.data.bin +0 -0
  183. data/spec/support/protos/google_unittest.bin +0 -0
  184. data/spec/support/protos/google_unittest.pb.rb +798 -0
  185. data/spec/support/protos/google_unittest.proto +884 -0
  186. data/spec/support/protos/google_unittest_custom_options.bin +0 -0
  187. data/spec/support/protos/google_unittest_custom_options.pb.rb +361 -0
  188. data/spec/support/protos/google_unittest_custom_options.proto +424 -0
  189. data/spec/support/protos/google_unittest_import.pb.rb +55 -0
  190. data/spec/support/protos/google_unittest_import.proto +73 -0
  191. data/spec/support/protos/google_unittest_import_public.pb.rb +31 -0
  192. data/spec/support/protos/google_unittest_import_public.proto +41 -0
  193. data/spec/support/protos/map-test.bin +157 -0
  194. data/spec/support/protos/map-test.pb.rb +85 -0
  195. data/spec/support/protos/map-test.proto +68 -0
  196. data/spec/support/protos/multi_field_extensions.pb.rb +59 -0
  197. data/spec/support/protos/multi_field_extensions.proto +35 -0
  198. data/spec/support/protos/resource.pb.rb +172 -0
  199. data/spec/support/protos/resource.proto +137 -0
  200. data/spec/support/resource_service.rb +23 -0
  201. data/spec/support/server.rb +65 -0
  202. data/spec/support/test_app_file.rb +2 -0
  203. data/varint_prof.rb +82 -0
  204. metadata +579 -0
@@ -0,0 +1,194 @@
1
+ require 'thread'
2
+
3
+ module Protobuf
4
+ module Rpc
5
+ module Zmq
6
+ class Broker
7
+ include ::Protobuf::Rpc::Zmq::Util
8
+
9
+ attr_reader :local_queue
10
+
11
+ def initialize(server)
12
+ @server = server
13
+
14
+ init_zmq_context
15
+ init_local_queue
16
+ init_backend_socket
17
+ init_frontend_socket
18
+ init_poller
19
+ rescue
20
+ teardown
21
+ raise
22
+ end
23
+
24
+ def run
25
+ @idle_workers = []
26
+ @running = true
27
+
28
+ loop do
29
+ process_local_queue
30
+ rc = @poller.poll(broker_polling_milliseconds)
31
+
32
+ # The server was shutdown and no requests are pending
33
+ break if rc == 0 && !running? && @server.workers.empty?
34
+ # Something went wrong
35
+ break if rc == -1
36
+
37
+ check_and_process_backend
38
+ process_local_queue # Fair ordering so queued requests get in before new requests
39
+ check_and_process_frontend
40
+ end
41
+ ensure
42
+ teardown
43
+ @running = false
44
+ end
45
+
46
+ def running?
47
+ @running && @server.running?
48
+ end
49
+
50
+ private
51
+
52
+ def backend_poll_weight
53
+ @backend_poll_weight ||= [ENV["PB_ZMQ_SERVER_BACKEND_POLL_WEIGHT"].to_i, 1].max
54
+ end
55
+
56
+ def broker_polling_milliseconds
57
+ @broker_polling_milliseconds ||= [ENV["PB_ZMQ_BROKER_POLLING_MILLISECONDS"].to_i, 500].max
58
+ end
59
+
60
+ def check_and_process_backend
61
+ readables_include_backend = @poller.readables.include?(@backend_socket)
62
+ message_count_read_from_backend = 0
63
+
64
+ while readables_include_backend && message_count_read_from_backend < backend_poll_weight
65
+ message_count_read_from_backend += 1
66
+ process_backend
67
+ @poller.poll_nonblock
68
+ readables_include_backend = @poller.readables.include?(@backend_socket)
69
+ end
70
+ end
71
+
72
+ def check_and_process_frontend
73
+ readables_include_frontend = @poller.readables.include?(@frontend_socket)
74
+ message_count_read_from_frontend = 0
75
+
76
+ while readables_include_frontend && message_count_read_from_frontend < frontend_poll_weight
77
+ message_count_read_from_frontend += 1
78
+ process_frontend
79
+ break unless local_queue_available? # no need to read frontend just to throw away messages, will prioritize backend when full
80
+ @poller.poll_nonblock
81
+ readables_include_frontend = @poller.readables.include?(@frontend_socket)
82
+ end
83
+ end
84
+
85
+ def frontend_poll_weight
86
+ @frontend_poll_weight ||= [ENV["PB_ZMQ_SERVER_FRONTEND_POLL_WEIGHT"].to_i, 1].max
87
+ end
88
+
89
+ def init_backend_socket
90
+ @backend_socket = @zmq_context.socket(ZMQ::ROUTER)
91
+ zmq_error_check(@backend_socket.bind(@server.backend_uri))
92
+ end
93
+
94
+ def init_frontend_socket
95
+ @frontend_socket = @zmq_context.socket(ZMQ::ROUTER)
96
+ zmq_error_check(@frontend_socket.bind(@server.frontend_uri))
97
+ end
98
+
99
+ def init_local_queue
100
+ @local_queue = []
101
+ end
102
+
103
+ def init_poller
104
+ @poller = ZMQ::Poller.new
105
+ @poller.register_readable(@frontend_socket)
106
+ @poller.register_readable(@backend_socket)
107
+ end
108
+
109
+ def init_zmq_context
110
+ @zmq_context =
111
+ if inproc?
112
+ @server.zmq_context
113
+ else
114
+ ZMQ::Context.new
115
+ end
116
+ end
117
+
118
+ def inproc?
119
+ !!@server.try(:inproc?)
120
+ end
121
+
122
+ def local_queue_available?
123
+ local_queue.size < local_queue_max_size && running?
124
+ end
125
+
126
+ def local_queue_max_size
127
+ @local_queue_max_size ||= [ENV["PB_ZMQ_SERVER_QUEUE_MAX_SIZE"].to_i, 5].max
128
+ end
129
+
130
+ def process_backend
131
+ worker, _ignore, *frames = read_from_backend
132
+
133
+ @idle_workers << worker
134
+
135
+ unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
136
+ write_to_frontend(frames)
137
+ end
138
+ end
139
+
140
+ def process_frontend
141
+ address, _, message, *frames = read_from_frontend
142
+
143
+ if message == ::Protobuf::Rpc::Zmq::CHECK_AVAILABLE_MESSAGE
144
+ if local_queue_available?
145
+ write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::WORKERS_AVAILABLE])
146
+ else
147
+ write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::NO_WORKERS_AVAILABLE])
148
+ end
149
+ else
150
+ if @idle_workers.empty? # rubocop:disable Style/IfInsideElse
151
+ local_queue << [address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message].concat(frames)
152
+ else
153
+ write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message]).concat(frames))
154
+ end
155
+ end
156
+ end
157
+
158
+ def process_local_queue
159
+ return if local_queue.empty?
160
+ return if @idle_workers.empty?
161
+
162
+ write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat(local_queue.shift))
163
+ process_local_queue
164
+ end
165
+
166
+ def read_from_backend
167
+ frames = []
168
+ zmq_error_check(@backend_socket.recv_strings(frames))
169
+ frames
170
+ end
171
+
172
+ def read_from_frontend
173
+ frames = []
174
+ zmq_error_check(@frontend_socket.recv_strings(frames))
175
+ frames
176
+ end
177
+
178
+ def teardown
179
+ @frontend_socket.try(:close)
180
+ @backend_socket.try(:close)
181
+ @zmq_context.try(:terminate) unless inproc?
182
+ end
183
+
184
+ def write_to_backend(frames)
185
+ zmq_error_check(@backend_socket.send_strings(frames))
186
+ end
187
+
188
+ def write_to_frontend(frames)
189
+ zmq_error_check(@frontend_socket.send_strings(frames))
190
+ end
191
+ end
192
+ end
193
+ end
194
+ end
@@ -0,0 +1,321 @@
1
+ require 'protobuf/rpc/servers/zmq/util'
2
+ require 'protobuf/rpc/servers/zmq/worker'
3
+ require 'protobuf/rpc/servers/zmq/broker'
4
+ require 'protobuf/rpc/dynamic_discovery.pb'
5
+ require 'securerandom'
6
+ require 'thread'
7
+
8
+ module Protobuf
9
+ module Rpc
10
+ module Zmq
11
+ class Server
12
+ include ::Protobuf::Rpc::Zmq::Util
13
+
14
+ DEFAULT_OPTIONS = {
15
+ :beacon_interval => 5,
16
+ :broadcast_beacons => false,
17
+ :broadcast_busy => false,
18
+ :zmq_inproc => true,
19
+ }.freeze
20
+
21
+ attr_accessor :options, :workers
22
+ attr_reader :zmq_context
23
+
24
+ def initialize(options)
25
+ @options = DEFAULT_OPTIONS.merge(options)
26
+ @workers = []
27
+
28
+ init_zmq_context
29
+ init_beacon_socket if broadcast_beacons?
30
+ init_shutdown_pipe
31
+ rescue
32
+ teardown
33
+ raise
34
+ end
35
+
36
+ def add_worker
37
+ @total_workers = total_workers + 1
38
+ end
39
+
40
+ def all_workers_busy?
41
+ workers.all? { |thread| !!thread[:busy] }
42
+ end
43
+
44
+ def backend_port
45
+ options[:worker_port] || frontend_port + 1
46
+ end
47
+
48
+ def backend_uri
49
+ if inproc?
50
+ "inproc://#{backend_ip}:#{backend_port}"
51
+ else
52
+ "tcp://#{backend_ip}:#{backend_port}"
53
+ end
54
+ end
55
+
56
+ def beacon_interval
57
+ [options[:beacon_interval].to_i, 1].max
58
+ end
59
+
60
+ def beacon_ip
61
+ "255.255.255.255"
62
+ end
63
+
64
+ def beacon_port
65
+ @beacon_port ||= options.fetch(
66
+ :beacon_port,
67
+ ::Protobuf::Rpc::ServiceDirectory.port,
68
+ ).to_i
69
+ end
70
+
71
+ def beacon_uri
72
+ "udp://#{beacon_ip}:#{beacon_port}"
73
+ end
74
+
75
+ def broadcast_beacons?
76
+ !brokerless? && options[:broadcast_beacons]
77
+ end
78
+
79
+ def broadcast_busy?
80
+ broadcast_beacons? && options[:broadcast_busy]
81
+ end
82
+
83
+ def broadcast_flatline
84
+ flatline = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
85
+ :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE,
86
+ :server => to_proto,
87
+ )
88
+
89
+ @beacon_socket.send(flatline.encode, 0)
90
+ end
91
+
92
+ def broadcast_heartbeat
93
+ @last_beacon = Time.now.to_i
94
+
95
+ heartbeat = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
96
+ :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT,
97
+ :server => to_proto,
98
+ )
99
+
100
+ @beacon_socket.send(heartbeat.encode, 0)
101
+
102
+ logger.debug { sign_message("sent heartbeat to #{beacon_uri}") }
103
+ end
104
+
105
+ def broadcast_heartbeat?
106
+ Time.now.to_i >= next_beacon && broadcast_beacons?
107
+ end
108
+
109
+ def brokerless?
110
+ !!options[:workers_only]
111
+ end
112
+
113
+ def busy_worker_count
114
+ workers.count { |thread| !!thread[:busy] }
115
+ end
116
+
117
+ def frontend_ip
118
+ @frontend_ip ||= resolve_ip(options[:host])
119
+ end
120
+ alias :backend_ip frontend_ip
121
+
122
+ def frontend_port
123
+ options[:port]
124
+ end
125
+
126
+ def frontend_uri
127
+ "tcp://#{frontend_ip}:#{frontend_port}"
128
+ end
129
+
130
+ def inproc?
131
+ !!options[:zmq_inproc]
132
+ end
133
+
134
+ def maintenance_timeout
135
+ next_maintenance - Time.now.to_i
136
+ end
137
+
138
+ def next_maintenance
139
+ cycles = [next_reaping]
140
+ cycles << next_beacon if broadcast_beacons?
141
+
142
+ cycles.min
143
+ end
144
+
145
+ def minimum_timeout
146
+ 0.1
147
+ end
148
+
149
+ def next_beacon
150
+ if @last_beacon.nil?
151
+ 0
152
+ else
153
+ @last_beacon + beacon_interval
154
+ end
155
+ end
156
+
157
+ def next_reaping
158
+ if @last_reaping.nil?
159
+ 0
160
+ else
161
+ @last_reaping + reaping_interval
162
+ end
163
+ end
164
+
165
+ def reap_dead_workers
166
+ @last_reaping = Time.now.to_i
167
+
168
+ @workers.keep_if do |worker|
169
+ worker.alive? || worker.join && false
170
+ end
171
+ end
172
+
173
+ def reap_dead_workers?
174
+ Time.now.to_i >= next_reaping
175
+ end
176
+
177
+ def reaping_interval
178
+ 5
179
+ end
180
+
181
+ def run
182
+ @running = true
183
+ yield if block_given? # runs on startup
184
+ wait_for_shutdown_signal
185
+ broadcast_flatline if broadcast_beacons?
186
+ Thread.pass until reap_dead_workers.empty?
187
+ @broker_thread.join unless brokerless?
188
+ ensure
189
+ @running = false
190
+ teardown
191
+ end
192
+
193
+ def running?
194
+ !!@running
195
+ end
196
+
197
+ def start_missing_workers
198
+ missing_workers = total_workers - @workers.size
199
+
200
+ if missing_workers > 0
201
+ missing_workers.times { start_worker }
202
+ logger.debug { sign_message("#{total_workers} workers started") }
203
+ end
204
+ end
205
+
206
+ def stop
207
+ @running = false
208
+ @shutdown_w.write('.')
209
+ end
210
+
211
+ def teardown
212
+ @shutdown_r.try(:close)
213
+ @shutdown_w.try(:close)
214
+ @beacon_socket.try(:close)
215
+ @zmq_context.try(:terminate)
216
+ @last_reaping = @last_beacon = @timeout = nil
217
+ end
218
+
219
+ def timeout
220
+ @timeout =
221
+ if @timeout.nil?
222
+ 0
223
+ else
224
+ [minimum_timeout, maintenance_timeout].max
225
+ end
226
+ end
227
+
228
+ def total_workers
229
+ @total_workers ||= [@options[:threads].to_i, 1].max
230
+ end
231
+
232
+ def to_proto
233
+ @proto ||= ::Protobuf::Rpc::DynamicDiscovery::Server.new(
234
+ :uuid => uuid,
235
+ :address => frontend_ip,
236
+ :port => frontend_port.to_s,
237
+ :ttl => (beacon_interval * 1.5).ceil,
238
+ :services => ::Protobuf::Rpc::Service.implemented_services,
239
+ )
240
+ end
241
+
242
+ def uuid
243
+ @uuid ||= SecureRandom.uuid
244
+ end
245
+
246
+ def wait_for_shutdown_signal
247
+ loop do
248
+ break if IO.select([@shutdown_r], nil, nil, timeout)
249
+
250
+ start_broker unless brokerless?
251
+ reap_dead_workers if reap_dead_workers?
252
+ start_missing_workers
253
+
254
+ next unless broadcast_heartbeat?
255
+
256
+ if broadcast_busy? && all_workers_busy?
257
+ broadcast_flatline
258
+ else
259
+ broadcast_heartbeat
260
+ end
261
+ end
262
+ end
263
+
264
+ private
265
+
266
+ def init_beacon_socket
267
+ @beacon_socket = UDPSocket.new
268
+ @beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_BROADCAST, true)
269
+ @beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
270
+
271
+ if defined?(::Socket::SO_REUSEPORT)
272
+ @beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, true)
273
+ end
274
+
275
+ @beacon_socket.bind(frontend_ip, beacon_port)
276
+ @beacon_socket.connect(beacon_ip, beacon_port)
277
+ end
278
+
279
+ def init_shutdown_pipe
280
+ @shutdown_r, @shutdown_w = IO.pipe
281
+ end
282
+
283
+ def init_zmq_context
284
+ @zmq_context = ZMQ::Context.new
285
+ end
286
+
287
+ def start_broker
288
+ return if @broker && @broker.running? && @broker_thread.alive?
289
+ if @broker && !@broker.running?
290
+ broadcast_flatline if broadcast_busy?
291
+ @broker_thread.join if @broker_thread
292
+ init_zmq_context # need a new context to restart the broker
293
+ end
294
+
295
+ @broker = ::Protobuf::Rpc::Zmq::Broker.new(self)
296
+ @broker_thread = Thread.new(@broker) do |broker|
297
+ begin
298
+ broker.run
299
+ rescue => e
300
+ message = "Broker failed: #{e.inspect}\n #{e.backtrace.join($INPUT_RECORD_SEPARATOR)}"
301
+ $stderr.puts(message)
302
+ logger.error { message }
303
+ end
304
+ end
305
+ end
306
+
307
+ def start_worker
308
+ @workers << Thread.new(self, @broker) do |server, broker|
309
+ begin
310
+ ::Protobuf::Rpc::Zmq::Worker.new(server, broker).run
311
+ rescue => e
312
+ message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($INPUT_RECORD_SEPARATOR)}"
313
+ $stderr.puts(message)
314
+ logger.error { message }
315
+ end
316
+ end
317
+ end
318
+ end
319
+ end
320
+ end
321
+ end