protobuf-cucumber 3.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +21 -0
- data/.rubocop.yml +70 -0
- data/.rubocop_todo.yml +145 -0
- data/.travis.yml +40 -0
- data/.yardopts +5 -0
- data/CHANGES.md +344 -0
- data/CONTRIBUTING.md +16 -0
- data/Gemfile +3 -0
- data/LICENSE.txt +22 -0
- data/README.md +33 -0
- data/Rakefile +64 -0
- data/bin/protoc-gen-ruby +22 -0
- data/bin/rpc_server +5 -0
- data/install-protobuf.sh +28 -0
- data/lib/protobuf.rb +129 -0
- data/lib/protobuf/cli.rb +257 -0
- data/lib/protobuf/code_generator.rb +120 -0
- data/lib/protobuf/decoder.rb +28 -0
- data/lib/protobuf/deprecation.rb +117 -0
- data/lib/protobuf/descriptors.rb +3 -0
- data/lib/protobuf/descriptors/google/protobuf/compiler/plugin.pb.rb +62 -0
- data/lib/protobuf/descriptors/google/protobuf/descriptor.pb.rb +301 -0
- data/lib/protobuf/encoder.rb +11 -0
- data/lib/protobuf/enum.rb +365 -0
- data/lib/protobuf/exceptions.rb +9 -0
- data/lib/protobuf/field.rb +74 -0
- data/lib/protobuf/field/base_field.rb +380 -0
- data/lib/protobuf/field/base_field_object_definitions.rb +504 -0
- data/lib/protobuf/field/bool_field.rb +64 -0
- data/lib/protobuf/field/bytes_field.rb +78 -0
- data/lib/protobuf/field/double_field.rb +25 -0
- data/lib/protobuf/field/enum_field.rb +61 -0
- data/lib/protobuf/field/field_array.rb +104 -0
- data/lib/protobuf/field/field_hash.rb +122 -0
- data/lib/protobuf/field/fixed32_field.rb +25 -0
- data/lib/protobuf/field/fixed64_field.rb +28 -0
- data/lib/protobuf/field/float_field.rb +43 -0
- data/lib/protobuf/field/int32_field.rb +21 -0
- data/lib/protobuf/field/int64_field.rb +34 -0
- data/lib/protobuf/field/integer_field.rb +23 -0
- data/lib/protobuf/field/message_field.rb +51 -0
- data/lib/protobuf/field/sfixed32_field.rb +27 -0
- data/lib/protobuf/field/sfixed64_field.rb +28 -0
- data/lib/protobuf/field/signed_integer_field.rb +29 -0
- data/lib/protobuf/field/sint32_field.rb +21 -0
- data/lib/protobuf/field/sint64_field.rb +21 -0
- data/lib/protobuf/field/string_field.rb +51 -0
- data/lib/protobuf/field/uint32_field.rb +21 -0
- data/lib/protobuf/field/uint64_field.rb +21 -0
- data/lib/protobuf/field/varint_field.rb +77 -0
- data/lib/protobuf/generators/base.rb +85 -0
- data/lib/protobuf/generators/enum_generator.rb +39 -0
- data/lib/protobuf/generators/extension_generator.rb +27 -0
- data/lib/protobuf/generators/field_generator.rb +193 -0
- data/lib/protobuf/generators/file_generator.rb +262 -0
- data/lib/protobuf/generators/group_generator.rb +122 -0
- data/lib/protobuf/generators/message_generator.rb +104 -0
- data/lib/protobuf/generators/option_generator.rb +17 -0
- data/lib/protobuf/generators/printable.rb +160 -0
- data/lib/protobuf/generators/service_generator.rb +50 -0
- data/lib/protobuf/lifecycle.rb +33 -0
- data/lib/protobuf/logging.rb +39 -0
- data/lib/protobuf/message.rb +260 -0
- data/lib/protobuf/message/fields.rb +233 -0
- data/lib/protobuf/message/serialization.rb +85 -0
- data/lib/protobuf/optionable.rb +70 -0
- data/lib/protobuf/rpc/buffer.rb +78 -0
- data/lib/protobuf/rpc/client.rb +140 -0
- data/lib/protobuf/rpc/connectors/base.rb +221 -0
- data/lib/protobuf/rpc/connectors/ping.rb +89 -0
- data/lib/protobuf/rpc/connectors/socket.rb +78 -0
- data/lib/protobuf/rpc/connectors/zmq.rb +319 -0
- data/lib/protobuf/rpc/dynamic_discovery.pb.rb +50 -0
- data/lib/protobuf/rpc/env.rb +60 -0
- data/lib/protobuf/rpc/error.rb +28 -0
- data/lib/protobuf/rpc/error/client_error.rb +31 -0
- data/lib/protobuf/rpc/error/server_error.rb +43 -0
- data/lib/protobuf/rpc/middleware.rb +25 -0
- data/lib/protobuf/rpc/middleware/exception_handler.rb +40 -0
- data/lib/protobuf/rpc/middleware/logger.rb +95 -0
- data/lib/protobuf/rpc/middleware/request_decoder.rb +79 -0
- data/lib/protobuf/rpc/middleware/response_encoder.rb +83 -0
- data/lib/protobuf/rpc/middleware/runner.rb +18 -0
- data/lib/protobuf/rpc/rpc.pb.rb +64 -0
- data/lib/protobuf/rpc/rpc_method.rb +16 -0
- data/lib/protobuf/rpc/server.rb +39 -0
- data/lib/protobuf/rpc/servers/socket/server.rb +121 -0
- data/lib/protobuf/rpc/servers/socket/worker.rb +56 -0
- data/lib/protobuf/rpc/servers/socket_runner.rb +46 -0
- data/lib/protobuf/rpc/servers/zmq/broker.rb +194 -0
- data/lib/protobuf/rpc/servers/zmq/server.rb +321 -0
- data/lib/protobuf/rpc/servers/zmq/util.rb +48 -0
- data/lib/protobuf/rpc/servers/zmq/worker.rb +105 -0
- data/lib/protobuf/rpc/servers/zmq_runner.rb +70 -0
- data/lib/protobuf/rpc/service.rb +172 -0
- data/lib/protobuf/rpc/service_directory.rb +261 -0
- data/lib/protobuf/rpc/service_dispatcher.rb +45 -0
- data/lib/protobuf/rpc/service_filters.rb +250 -0
- data/lib/protobuf/rpc/stat.rb +119 -0
- data/lib/protobuf/socket.rb +21 -0
- data/lib/protobuf/tasks.rb +1 -0
- data/lib/protobuf/tasks/compile.rake +80 -0
- data/lib/protobuf/varint.rb +20 -0
- data/lib/protobuf/varint_pure.rb +31 -0
- data/lib/protobuf/version.rb +3 -0
- data/lib/protobuf/wire_type.rb +10 -0
- data/lib/protobuf/zmq.rb +21 -0
- data/profile.html +5103 -0
- data/proto/dynamic_discovery.proto +44 -0
- data/proto/google/protobuf/compiler/plugin.proto +147 -0
- data/proto/google/protobuf/descriptor.proto +779 -0
- data/proto/rpc.proto +69 -0
- data/protobuf-cucumber.gemspec +57 -0
- data/spec/benchmark/tasks.rb +143 -0
- data/spec/bin/protoc-gen-ruby_spec.rb +23 -0
- data/spec/encoding/all_types_spec.rb +103 -0
- data/spec/encoding/extreme_values_spec.rb +0 -0
- data/spec/functional/class_inheritance_spec.rb +52 -0
- data/spec/functional/code_generator_spec.rb +58 -0
- data/spec/functional/socket_server_spec.rb +59 -0
- data/spec/functional/zmq_server_spec.rb +105 -0
- data/spec/lib/protobuf/cli_spec.rb +317 -0
- data/spec/lib/protobuf/code_generator_spec.rb +87 -0
- data/spec/lib/protobuf/enum_spec.rb +307 -0
- data/spec/lib/protobuf/field/bool_field_spec.rb +91 -0
- data/spec/lib/protobuf/field/double_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/enum_field_spec.rb +44 -0
- data/spec/lib/protobuf/field/field_array_spec.rb +105 -0
- data/spec/lib/protobuf/field/field_hash_spec.rb +168 -0
- data/spec/lib/protobuf/field/fixed32_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/fixed64_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/float_field_spec.rb +90 -0
- data/spec/lib/protobuf/field/int32_field_spec.rb +120 -0
- data/spec/lib/protobuf/field/int64_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/message_field_spec.rb +132 -0
- data/spec/lib/protobuf/field/sfixed32_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/sfixed64_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/sint32_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/sint64_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/string_field_spec.rb +79 -0
- data/spec/lib/protobuf/field/uint32_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/uint64_field_spec.rb +7 -0
- data/spec/lib/protobuf/field_spec.rb +192 -0
- data/spec/lib/protobuf/generators/base_spec.rb +154 -0
- data/spec/lib/protobuf/generators/enum_generator_spec.rb +82 -0
- data/spec/lib/protobuf/generators/extension_generator_spec.rb +42 -0
- data/spec/lib/protobuf/generators/field_generator_spec.rb +197 -0
- data/spec/lib/protobuf/generators/file_generator_spec.rb +119 -0
- data/spec/lib/protobuf/generators/message_generator_spec.rb +0 -0
- data/spec/lib/protobuf/generators/service_generator_spec.rb +99 -0
- data/spec/lib/protobuf/lifecycle_spec.rb +94 -0
- data/spec/lib/protobuf/message_spec.rb +944 -0
- data/spec/lib/protobuf/optionable_spec.rb +265 -0
- data/spec/lib/protobuf/rpc/client_spec.rb +66 -0
- data/spec/lib/protobuf/rpc/connectors/base_spec.rb +226 -0
- data/spec/lib/protobuf/rpc/connectors/ping_spec.rb +69 -0
- data/spec/lib/protobuf/rpc/connectors/socket_spec.rb +34 -0
- data/spec/lib/protobuf/rpc/connectors/zmq_spec.rb +110 -0
- data/spec/lib/protobuf/rpc/middleware/exception_handler_spec.rb +62 -0
- data/spec/lib/protobuf/rpc/middleware/logger_spec.rb +49 -0
- data/spec/lib/protobuf/rpc/middleware/request_decoder_spec.rb +115 -0
- data/spec/lib/protobuf/rpc/middleware/response_encoder_spec.rb +91 -0
- data/spec/lib/protobuf/rpc/servers/socket_server_spec.rb +38 -0
- data/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb +43 -0
- data/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb +55 -0
- data/spec/lib/protobuf/rpc/servers/zmq/worker_spec.rb +35 -0
- data/spec/lib/protobuf/rpc/service_directory_spec.rb +293 -0
- data/spec/lib/protobuf/rpc/service_dispatcher_spec.rb +35 -0
- data/spec/lib/protobuf/rpc/service_filters_spec.rb +517 -0
- data/spec/lib/protobuf/rpc/service_spec.rb +162 -0
- data/spec/lib/protobuf/rpc/stat_spec.rb +101 -0
- data/spec/lib/protobuf/varint_spec.rb +29 -0
- data/spec/lib/protobuf_spec.rb +105 -0
- data/spec/spec_helper.rb +42 -0
- data/spec/support/all.rb +6 -0
- data/spec/support/packed_field.rb +23 -0
- data/spec/support/protos/all_types.data.bin +0 -0
- data/spec/support/protos/all_types.data.txt +119 -0
- data/spec/support/protos/enum.pb.rb +63 -0
- data/spec/support/protos/enum.proto +37 -0
- data/spec/support/protos/extreme_values.data.bin +0 -0
- data/spec/support/protos/google_unittest.bin +0 -0
- data/spec/support/protos/google_unittest.pb.rb +798 -0
- data/spec/support/protos/google_unittest.proto +884 -0
- data/spec/support/protos/google_unittest_custom_options.bin +0 -0
- data/spec/support/protos/google_unittest_custom_options.pb.rb +361 -0
- data/spec/support/protos/google_unittest_custom_options.proto +424 -0
- data/spec/support/protos/google_unittest_import.pb.rb +55 -0
- data/spec/support/protos/google_unittest_import.proto +73 -0
- data/spec/support/protos/google_unittest_import_public.pb.rb +31 -0
- data/spec/support/protos/google_unittest_import_public.proto +41 -0
- data/spec/support/protos/map-test.bin +157 -0
- data/spec/support/protos/map-test.pb.rb +85 -0
- data/spec/support/protos/map-test.proto +68 -0
- data/spec/support/protos/multi_field_extensions.pb.rb +59 -0
- data/spec/support/protos/multi_field_extensions.proto +35 -0
- data/spec/support/protos/resource.pb.rb +172 -0
- data/spec/support/protos/resource.proto +137 -0
- data/spec/support/resource_service.rb +23 -0
- data/spec/support/server.rb +65 -0
- data/spec/support/test_app_file.rb +2 -0
- data/varint_prof.rb +82 -0
- metadata +579 -0
@@ -0,0 +1,194 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module Protobuf
|
4
|
+
module Rpc
|
5
|
+
module Zmq
|
6
|
+
class Broker
|
7
|
+
include ::Protobuf::Rpc::Zmq::Util
|
8
|
+
|
9
|
+
attr_reader :local_queue
|
10
|
+
|
11
|
+
def initialize(server)
|
12
|
+
@server = server
|
13
|
+
|
14
|
+
init_zmq_context
|
15
|
+
init_local_queue
|
16
|
+
init_backend_socket
|
17
|
+
init_frontend_socket
|
18
|
+
init_poller
|
19
|
+
rescue
|
20
|
+
teardown
|
21
|
+
raise
|
22
|
+
end
|
23
|
+
|
24
|
+
def run
|
25
|
+
@idle_workers = []
|
26
|
+
@running = true
|
27
|
+
|
28
|
+
loop do
|
29
|
+
process_local_queue
|
30
|
+
rc = @poller.poll(broker_polling_milliseconds)
|
31
|
+
|
32
|
+
# The server was shutdown and no requests are pending
|
33
|
+
break if rc == 0 && !running? && @server.workers.empty?
|
34
|
+
# Something went wrong
|
35
|
+
break if rc == -1
|
36
|
+
|
37
|
+
check_and_process_backend
|
38
|
+
process_local_queue # Fair ordering so queued requests get in before new requests
|
39
|
+
check_and_process_frontend
|
40
|
+
end
|
41
|
+
ensure
|
42
|
+
teardown
|
43
|
+
@running = false
|
44
|
+
end
|
45
|
+
|
46
|
+
def running?
|
47
|
+
@running && @server.running?
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def backend_poll_weight
|
53
|
+
@backend_poll_weight ||= [ENV["PB_ZMQ_SERVER_BACKEND_POLL_WEIGHT"].to_i, 1].max
|
54
|
+
end
|
55
|
+
|
56
|
+
def broker_polling_milliseconds
|
57
|
+
@broker_polling_milliseconds ||= [ENV["PB_ZMQ_BROKER_POLLING_MILLISECONDS"].to_i, 500].max
|
58
|
+
end
|
59
|
+
|
60
|
+
def check_and_process_backend
|
61
|
+
readables_include_backend = @poller.readables.include?(@backend_socket)
|
62
|
+
message_count_read_from_backend = 0
|
63
|
+
|
64
|
+
while readables_include_backend && message_count_read_from_backend < backend_poll_weight
|
65
|
+
message_count_read_from_backend += 1
|
66
|
+
process_backend
|
67
|
+
@poller.poll_nonblock
|
68
|
+
readables_include_backend = @poller.readables.include?(@backend_socket)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def check_and_process_frontend
|
73
|
+
readables_include_frontend = @poller.readables.include?(@frontend_socket)
|
74
|
+
message_count_read_from_frontend = 0
|
75
|
+
|
76
|
+
while readables_include_frontend && message_count_read_from_frontend < frontend_poll_weight
|
77
|
+
message_count_read_from_frontend += 1
|
78
|
+
process_frontend
|
79
|
+
break unless local_queue_available? # no need to read frontend just to throw away messages, will prioritize backend when full
|
80
|
+
@poller.poll_nonblock
|
81
|
+
readables_include_frontend = @poller.readables.include?(@frontend_socket)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def frontend_poll_weight
|
86
|
+
@frontend_poll_weight ||= [ENV["PB_ZMQ_SERVER_FRONTEND_POLL_WEIGHT"].to_i, 1].max
|
87
|
+
end
|
88
|
+
|
89
|
+
def init_backend_socket
|
90
|
+
@backend_socket = @zmq_context.socket(ZMQ::ROUTER)
|
91
|
+
zmq_error_check(@backend_socket.bind(@server.backend_uri))
|
92
|
+
end
|
93
|
+
|
94
|
+
def init_frontend_socket
|
95
|
+
@frontend_socket = @zmq_context.socket(ZMQ::ROUTER)
|
96
|
+
zmq_error_check(@frontend_socket.bind(@server.frontend_uri))
|
97
|
+
end
|
98
|
+
|
99
|
+
def init_local_queue
|
100
|
+
@local_queue = []
|
101
|
+
end
|
102
|
+
|
103
|
+
def init_poller
|
104
|
+
@poller = ZMQ::Poller.new
|
105
|
+
@poller.register_readable(@frontend_socket)
|
106
|
+
@poller.register_readable(@backend_socket)
|
107
|
+
end
|
108
|
+
|
109
|
+
def init_zmq_context
|
110
|
+
@zmq_context =
|
111
|
+
if inproc?
|
112
|
+
@server.zmq_context
|
113
|
+
else
|
114
|
+
ZMQ::Context.new
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def inproc?
|
119
|
+
!!@server.try(:inproc?)
|
120
|
+
end
|
121
|
+
|
122
|
+
def local_queue_available?
|
123
|
+
local_queue.size < local_queue_max_size && running?
|
124
|
+
end
|
125
|
+
|
126
|
+
def local_queue_max_size
|
127
|
+
@local_queue_max_size ||= [ENV["PB_ZMQ_SERVER_QUEUE_MAX_SIZE"].to_i, 5].max
|
128
|
+
end
|
129
|
+
|
130
|
+
def process_backend
|
131
|
+
worker, _ignore, *frames = read_from_backend
|
132
|
+
|
133
|
+
@idle_workers << worker
|
134
|
+
|
135
|
+
unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
|
136
|
+
write_to_frontend(frames)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def process_frontend
|
141
|
+
address, _, message, *frames = read_from_frontend
|
142
|
+
|
143
|
+
if message == ::Protobuf::Rpc::Zmq::CHECK_AVAILABLE_MESSAGE
|
144
|
+
if local_queue_available?
|
145
|
+
write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::WORKERS_AVAILABLE])
|
146
|
+
else
|
147
|
+
write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::NO_WORKERS_AVAILABLE])
|
148
|
+
end
|
149
|
+
else
|
150
|
+
if @idle_workers.empty? # rubocop:disable Style/IfInsideElse
|
151
|
+
local_queue << [address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message].concat(frames)
|
152
|
+
else
|
153
|
+
write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message]).concat(frames))
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def process_local_queue
|
159
|
+
return if local_queue.empty?
|
160
|
+
return if @idle_workers.empty?
|
161
|
+
|
162
|
+
write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat(local_queue.shift))
|
163
|
+
process_local_queue
|
164
|
+
end
|
165
|
+
|
166
|
+
def read_from_backend
|
167
|
+
frames = []
|
168
|
+
zmq_error_check(@backend_socket.recv_strings(frames))
|
169
|
+
frames
|
170
|
+
end
|
171
|
+
|
172
|
+
def read_from_frontend
|
173
|
+
frames = []
|
174
|
+
zmq_error_check(@frontend_socket.recv_strings(frames))
|
175
|
+
frames
|
176
|
+
end
|
177
|
+
|
178
|
+
def teardown
|
179
|
+
@frontend_socket.try(:close)
|
180
|
+
@backend_socket.try(:close)
|
181
|
+
@zmq_context.try(:terminate) unless inproc?
|
182
|
+
end
|
183
|
+
|
184
|
+
def write_to_backend(frames)
|
185
|
+
zmq_error_check(@backend_socket.send_strings(frames))
|
186
|
+
end
|
187
|
+
|
188
|
+
def write_to_frontend(frames)
|
189
|
+
zmq_error_check(@frontend_socket.send_strings(frames))
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
@@ -0,0 +1,321 @@
|
|
1
|
+
require 'protobuf/rpc/servers/zmq/util'
|
2
|
+
require 'protobuf/rpc/servers/zmq/worker'
|
3
|
+
require 'protobuf/rpc/servers/zmq/broker'
|
4
|
+
require 'protobuf/rpc/dynamic_discovery.pb'
|
5
|
+
require 'securerandom'
|
6
|
+
require 'thread'
|
7
|
+
|
8
|
+
module Protobuf
|
9
|
+
module Rpc
|
10
|
+
module Zmq
|
11
|
+
class Server
|
12
|
+
include ::Protobuf::Rpc::Zmq::Util
|
13
|
+
|
14
|
+
DEFAULT_OPTIONS = {
|
15
|
+
:beacon_interval => 5,
|
16
|
+
:broadcast_beacons => false,
|
17
|
+
:broadcast_busy => false,
|
18
|
+
:zmq_inproc => true,
|
19
|
+
}.freeze
|
20
|
+
|
21
|
+
attr_accessor :options, :workers
|
22
|
+
attr_reader :zmq_context
|
23
|
+
|
24
|
+
def initialize(options)
|
25
|
+
@options = DEFAULT_OPTIONS.merge(options)
|
26
|
+
@workers = []
|
27
|
+
|
28
|
+
init_zmq_context
|
29
|
+
init_beacon_socket if broadcast_beacons?
|
30
|
+
init_shutdown_pipe
|
31
|
+
rescue
|
32
|
+
teardown
|
33
|
+
raise
|
34
|
+
end
|
35
|
+
|
36
|
+
def add_worker
|
37
|
+
@total_workers = total_workers + 1
|
38
|
+
end
|
39
|
+
|
40
|
+
def all_workers_busy?
|
41
|
+
workers.all? { |thread| !!thread[:busy] }
|
42
|
+
end
|
43
|
+
|
44
|
+
def backend_port
|
45
|
+
options[:worker_port] || frontend_port + 1
|
46
|
+
end
|
47
|
+
|
48
|
+
def backend_uri
|
49
|
+
if inproc?
|
50
|
+
"inproc://#{backend_ip}:#{backend_port}"
|
51
|
+
else
|
52
|
+
"tcp://#{backend_ip}:#{backend_port}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def beacon_interval
|
57
|
+
[options[:beacon_interval].to_i, 1].max
|
58
|
+
end
|
59
|
+
|
60
|
+
def beacon_ip
|
61
|
+
"255.255.255.255"
|
62
|
+
end
|
63
|
+
|
64
|
+
def beacon_port
|
65
|
+
@beacon_port ||= options.fetch(
|
66
|
+
:beacon_port,
|
67
|
+
::Protobuf::Rpc::ServiceDirectory.port,
|
68
|
+
).to_i
|
69
|
+
end
|
70
|
+
|
71
|
+
def beacon_uri
|
72
|
+
"udp://#{beacon_ip}:#{beacon_port}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def broadcast_beacons?
|
76
|
+
!brokerless? && options[:broadcast_beacons]
|
77
|
+
end
|
78
|
+
|
79
|
+
def broadcast_busy?
|
80
|
+
broadcast_beacons? && options[:broadcast_busy]
|
81
|
+
end
|
82
|
+
|
83
|
+
def broadcast_flatline
|
84
|
+
flatline = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
|
85
|
+
:beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE,
|
86
|
+
:server => to_proto,
|
87
|
+
)
|
88
|
+
|
89
|
+
@beacon_socket.send(flatline.encode, 0)
|
90
|
+
end
|
91
|
+
|
92
|
+
def broadcast_heartbeat
|
93
|
+
@last_beacon = Time.now.to_i
|
94
|
+
|
95
|
+
heartbeat = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
|
96
|
+
:beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT,
|
97
|
+
:server => to_proto,
|
98
|
+
)
|
99
|
+
|
100
|
+
@beacon_socket.send(heartbeat.encode, 0)
|
101
|
+
|
102
|
+
logger.debug { sign_message("sent heartbeat to #{beacon_uri}") }
|
103
|
+
end
|
104
|
+
|
105
|
+
def broadcast_heartbeat?
|
106
|
+
Time.now.to_i >= next_beacon && broadcast_beacons?
|
107
|
+
end
|
108
|
+
|
109
|
+
def brokerless?
|
110
|
+
!!options[:workers_only]
|
111
|
+
end
|
112
|
+
|
113
|
+
def busy_worker_count
|
114
|
+
workers.count { |thread| !!thread[:busy] }
|
115
|
+
end
|
116
|
+
|
117
|
+
def frontend_ip
|
118
|
+
@frontend_ip ||= resolve_ip(options[:host])
|
119
|
+
end
|
120
|
+
alias :backend_ip frontend_ip
|
121
|
+
|
122
|
+
def frontend_port
|
123
|
+
options[:port]
|
124
|
+
end
|
125
|
+
|
126
|
+
def frontend_uri
|
127
|
+
"tcp://#{frontend_ip}:#{frontend_port}"
|
128
|
+
end
|
129
|
+
|
130
|
+
def inproc?
|
131
|
+
!!options[:zmq_inproc]
|
132
|
+
end
|
133
|
+
|
134
|
+
def maintenance_timeout
|
135
|
+
next_maintenance - Time.now.to_i
|
136
|
+
end
|
137
|
+
|
138
|
+
def next_maintenance
|
139
|
+
cycles = [next_reaping]
|
140
|
+
cycles << next_beacon if broadcast_beacons?
|
141
|
+
|
142
|
+
cycles.min
|
143
|
+
end
|
144
|
+
|
145
|
+
def minimum_timeout
|
146
|
+
0.1
|
147
|
+
end
|
148
|
+
|
149
|
+
def next_beacon
|
150
|
+
if @last_beacon.nil?
|
151
|
+
0
|
152
|
+
else
|
153
|
+
@last_beacon + beacon_interval
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def next_reaping
|
158
|
+
if @last_reaping.nil?
|
159
|
+
0
|
160
|
+
else
|
161
|
+
@last_reaping + reaping_interval
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
def reap_dead_workers
|
166
|
+
@last_reaping = Time.now.to_i
|
167
|
+
|
168
|
+
@workers.keep_if do |worker|
|
169
|
+
worker.alive? || worker.join && false
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def reap_dead_workers?
|
174
|
+
Time.now.to_i >= next_reaping
|
175
|
+
end
|
176
|
+
|
177
|
+
def reaping_interval
|
178
|
+
5
|
179
|
+
end
|
180
|
+
|
181
|
+
def run
|
182
|
+
@running = true
|
183
|
+
yield if block_given? # runs on startup
|
184
|
+
wait_for_shutdown_signal
|
185
|
+
broadcast_flatline if broadcast_beacons?
|
186
|
+
Thread.pass until reap_dead_workers.empty?
|
187
|
+
@broker_thread.join unless brokerless?
|
188
|
+
ensure
|
189
|
+
@running = false
|
190
|
+
teardown
|
191
|
+
end
|
192
|
+
|
193
|
+
def running?
|
194
|
+
!!@running
|
195
|
+
end
|
196
|
+
|
197
|
+
def start_missing_workers
|
198
|
+
missing_workers = total_workers - @workers.size
|
199
|
+
|
200
|
+
if missing_workers > 0
|
201
|
+
missing_workers.times { start_worker }
|
202
|
+
logger.debug { sign_message("#{total_workers} workers started") }
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def stop
|
207
|
+
@running = false
|
208
|
+
@shutdown_w.write('.')
|
209
|
+
end
|
210
|
+
|
211
|
+
def teardown
|
212
|
+
@shutdown_r.try(:close)
|
213
|
+
@shutdown_w.try(:close)
|
214
|
+
@beacon_socket.try(:close)
|
215
|
+
@zmq_context.try(:terminate)
|
216
|
+
@last_reaping = @last_beacon = @timeout = nil
|
217
|
+
end
|
218
|
+
|
219
|
+
def timeout
|
220
|
+
@timeout =
|
221
|
+
if @timeout.nil?
|
222
|
+
0
|
223
|
+
else
|
224
|
+
[minimum_timeout, maintenance_timeout].max
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def total_workers
|
229
|
+
@total_workers ||= [@options[:threads].to_i, 1].max
|
230
|
+
end
|
231
|
+
|
232
|
+
def to_proto
|
233
|
+
@proto ||= ::Protobuf::Rpc::DynamicDiscovery::Server.new(
|
234
|
+
:uuid => uuid,
|
235
|
+
:address => frontend_ip,
|
236
|
+
:port => frontend_port.to_s,
|
237
|
+
:ttl => (beacon_interval * 1.5).ceil,
|
238
|
+
:services => ::Protobuf::Rpc::Service.implemented_services,
|
239
|
+
)
|
240
|
+
end
|
241
|
+
|
242
|
+
def uuid
|
243
|
+
@uuid ||= SecureRandom.uuid
|
244
|
+
end
|
245
|
+
|
246
|
+
def wait_for_shutdown_signal
|
247
|
+
loop do
|
248
|
+
break if IO.select([@shutdown_r], nil, nil, timeout)
|
249
|
+
|
250
|
+
start_broker unless brokerless?
|
251
|
+
reap_dead_workers if reap_dead_workers?
|
252
|
+
start_missing_workers
|
253
|
+
|
254
|
+
next unless broadcast_heartbeat?
|
255
|
+
|
256
|
+
if broadcast_busy? && all_workers_busy?
|
257
|
+
broadcast_flatline
|
258
|
+
else
|
259
|
+
broadcast_heartbeat
|
260
|
+
end
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
private
|
265
|
+
|
266
|
+
def init_beacon_socket
|
267
|
+
@beacon_socket = UDPSocket.new
|
268
|
+
@beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_BROADCAST, true)
|
269
|
+
@beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
|
270
|
+
|
271
|
+
if defined?(::Socket::SO_REUSEPORT)
|
272
|
+
@beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEPORT, true)
|
273
|
+
end
|
274
|
+
|
275
|
+
@beacon_socket.bind(frontend_ip, beacon_port)
|
276
|
+
@beacon_socket.connect(beacon_ip, beacon_port)
|
277
|
+
end
|
278
|
+
|
279
|
+
def init_shutdown_pipe
|
280
|
+
@shutdown_r, @shutdown_w = IO.pipe
|
281
|
+
end
|
282
|
+
|
283
|
+
def init_zmq_context
|
284
|
+
@zmq_context = ZMQ::Context.new
|
285
|
+
end
|
286
|
+
|
287
|
+
def start_broker
|
288
|
+
return if @broker && @broker.running? && @broker_thread.alive?
|
289
|
+
if @broker && !@broker.running?
|
290
|
+
broadcast_flatline if broadcast_busy?
|
291
|
+
@broker_thread.join if @broker_thread
|
292
|
+
init_zmq_context # need a new context to restart the broker
|
293
|
+
end
|
294
|
+
|
295
|
+
@broker = ::Protobuf::Rpc::Zmq::Broker.new(self)
|
296
|
+
@broker_thread = Thread.new(@broker) do |broker|
|
297
|
+
begin
|
298
|
+
broker.run
|
299
|
+
rescue => e
|
300
|
+
message = "Broker failed: #{e.inspect}\n #{e.backtrace.join($INPUT_RECORD_SEPARATOR)}"
|
301
|
+
$stderr.puts(message)
|
302
|
+
logger.error { message }
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
306
|
+
|
307
|
+
def start_worker
|
308
|
+
@workers << Thread.new(self, @broker) do |server, broker|
|
309
|
+
begin
|
310
|
+
::Protobuf::Rpc::Zmq::Worker.new(server, broker).run
|
311
|
+
rescue => e
|
312
|
+
message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($INPUT_RECORD_SEPARATOR)}"
|
313
|
+
$stderr.puts(message)
|
314
|
+
logger.error { message }
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
end
|
320
|
+
end
|
321
|
+
end
|