protobuffy 3.6.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +67 -0
- data/.rubocop_todo.yml +145 -0
- data/.travis.yml +25 -5
- data/CHANGES.md +55 -0
- data/CONTRIBUTING.md +1 -1
- data/LICENSE.txt +17 -9
- data/README.md +13 -12
- data/Rakefile +15 -11
- data/bin/protoc-gen-ruby +8 -3
- data/bin/rpc_server +1 -0
- data/examples/lib/example/reverse-client.rb +2 -2
- data/install-protobuf.sh +28 -0
- data/lib/protobuf.rb +57 -53
- data/lib/protobuf/cli.rb +94 -74
- data/lib/protobuf/code_generator.rb +60 -9
- data/lib/protobuf/decoder.rb +19 -65
- data/lib/protobuf/deprecation.rb +117 -0
- data/lib/protobuf/descriptors/google/protobuf/compiler/plugin.pb.rb +11 -1
- data/lib/protobuf/descriptors/google/protobuf/descriptor.pb.rb +55 -3
- data/lib/protobuf/encoder.rb +13 -53
- data/lib/protobuf/enum.rb +58 -63
- data/lib/protobuf/field.rb +4 -4
- data/lib/protobuf/field/base_field.rb +101 -173
- data/lib/protobuf/field/bool_field.rb +17 -11
- data/lib/protobuf/field/bytes_field.rb +21 -35
- data/lib/protobuf/field/double_field.rb +0 -1
- data/lib/protobuf/field/enum_field.rb +23 -22
- data/lib/protobuf/field/field_array.rb +5 -4
- data/lib/protobuf/field/fixed32_field.rb +1 -1
- data/lib/protobuf/field/fixed64_field.rb +0 -1
- data/lib/protobuf/field/float_field.rb +4 -1
- data/lib/protobuf/field/int32_field.rb +0 -1
- data/lib/protobuf/field/int64_field.rb +0 -1
- data/lib/protobuf/field/integer_field.rb +0 -1
- data/lib/protobuf/field/message_field.rb +13 -28
- data/lib/protobuf/field/sfixed32_field.rb +0 -1
- data/lib/protobuf/field/sfixed64_field.rb +0 -1
- data/lib/protobuf/field/signed_integer_field.rb +0 -1
- data/lib/protobuf/field/sint32_field.rb +0 -1
- data/lib/protobuf/field/sint64_field.rb +0 -1
- data/lib/protobuf/field/string_field.rb +2 -4
- data/lib/protobuf/field/uint32_field.rb +0 -1
- data/lib/protobuf/field/uint64_field.rb +0 -1
- data/lib/protobuf/field/varint_field.rb +30 -13
- data/lib/protobuf/generators/base.rb +30 -16
- data/lib/protobuf/generators/enum_generator.rb +6 -9
- data/lib/protobuf/generators/extension_generator.rb +1 -2
- data/lib/protobuf/generators/field_generator.rb +25 -13
- data/lib/protobuf/generators/file_generator.rb +157 -35
- data/lib/protobuf/generators/group_generator.rb +22 -17
- data/lib/protobuf/generators/message_generator.rb +13 -14
- data/lib/protobuf/generators/option_generator.rb +17 -0
- data/lib/protobuf/generators/printable.rb +12 -13
- data/lib/protobuf/generators/service_generator.rb +2 -3
- data/lib/protobuf/http.rb +2 -2
- data/lib/protobuf/lifecycle.rb +20 -33
- data/lib/protobuf/logging.rb +39 -0
- data/lib/protobuf/message.rb +114 -47
- data/lib/protobuf/message/fields.rb +170 -88
- data/lib/protobuf/message/serialization.rb +19 -18
- data/lib/protobuf/optionable.rb +53 -6
- data/lib/protobuf/rpc/buffer.rb +18 -19
- data/lib/protobuf/rpc/client.rb +22 -50
- data/lib/protobuf/rpc/connectors/base.rb +177 -12
- data/lib/protobuf/rpc/connectors/http.rb +14 -9
- data/lib/protobuf/rpc/connectors/ping.rb +89 -0
- data/lib/protobuf/rpc/connectors/socket.rb +13 -8
- data/lib/protobuf/rpc/connectors/zmq.rb +178 -73
- data/lib/protobuf/rpc/dynamic_discovery.pb.rb +4 -1
- data/lib/protobuf/rpc/env.rb +12 -12
- data/lib/protobuf/rpc/error.rb +3 -3
- data/lib/protobuf/rpc/error/client_error.rb +4 -4
- data/lib/protobuf/rpc/error/server_error.rb +9 -9
- data/lib/protobuf/rpc/middleware/exception_handler.rb +6 -2
- data/lib/protobuf/rpc/middleware/logger.rb +8 -4
- data/lib/protobuf/rpc/middleware/request_decoder.rb +17 -21
- data/lib/protobuf/rpc/middleware/response_encoder.rb +22 -27
- data/lib/protobuf/rpc/middleware/statsd.rb +3 -3
- data/lib/protobuf/rpc/rpc.pb.rb +4 -1
- data/lib/protobuf/rpc/server.rb +1 -1
- data/lib/protobuf/rpc/servers/http/server.rb +19 -17
- data/lib/protobuf/rpc/servers/socket/server.rb +78 -70
- data/lib/protobuf/rpc/servers/socket/worker.rb +4 -4
- data/lib/protobuf/rpc/servers/socket_runner.rb +27 -15
- data/lib/protobuf/rpc/servers/zmq/broker.rb +70 -31
- data/lib/protobuf/rpc/servers/zmq/server.rb +55 -47
- data/lib/protobuf/rpc/servers/zmq/util.rb +14 -13
- data/lib/protobuf/rpc/servers/zmq/worker.rb +16 -16
- data/lib/protobuf/rpc/servers/zmq_runner.rb +26 -7
- data/lib/protobuf/rpc/service.rb +21 -27
- data/lib/protobuf/rpc/service_directory.rb +43 -27
- data/lib/protobuf/rpc/service_dispatcher.rb +9 -10
- data/lib/protobuf/rpc/service_filters.rb +32 -55
- data/lib/protobuf/rpc/stat.rb +4 -8
- data/lib/protobuf/socket.rb +1 -2
- data/lib/protobuf/tasks/compile.rake +3 -4
- data/lib/protobuf/varint.rb +9 -0
- data/lib/protobuf/varint_pure.rb +13 -0
- data/lib/protobuf/version.rb +1 -1
- data/lib/protobuf/zmq.rb +2 -2
- data/proto/google/protobuf/descriptor.proto +190 -31
- data/protobuffy.gemspec +30 -17
- data/spec/benchmark/tasks.rb +27 -19
- data/spec/bin/protoc-gen-ruby_spec.rb +11 -6
- data/spec/encoding/all_types_spec.rb +96 -84
- data/spec/encoding/extreme_values_spec.rb +0 -0
- data/spec/functional/class_inheritance_spec.rb +52 -0
- data/spec/functional/code_generator_spec.rb +38 -0
- data/spec/functional/socket_server_spec.rb +15 -15
- data/spec/functional/zmq_server_spec.rb +29 -27
- data/spec/lib/protobuf/cli_spec.rb +82 -67
- data/spec/lib/protobuf/code_generator_spec.rb +37 -10
- data/spec/lib/protobuf/enum_spec.rb +77 -46
- data/spec/lib/protobuf/field/bool_field_spec.rb +91 -0
- data/spec/lib/protobuf/field/double_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/enum_field_spec.rb +26 -0
- data/spec/lib/protobuf/field/field_array_spec.rb +69 -0
- data/spec/lib/protobuf/field/fixed32_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/fixed64_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/float_field_spec.rb +90 -0
- data/spec/lib/protobuf/field/int32_field_spec.rb +114 -1
- data/spec/lib/protobuf/field/int64_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/message_field_spec.rb +132 -0
- data/spec/lib/protobuf/field/sfixed32_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/sfixed64_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/sint32_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/sint64_field_spec.rb +9 -0
- data/spec/lib/protobuf/field/string_field_spec.rb +44 -11
- data/spec/lib/protobuf/field/uint32_field_spec.rb +7 -0
- data/spec/lib/protobuf/field/uint64_field_spec.rb +7 -0
- data/spec/lib/protobuf/field_spec.rb +4 -6
- data/spec/lib/protobuf/generators/base_spec.rb +80 -13
- data/spec/lib/protobuf/generators/enum_generator_spec.rb +35 -21
- data/spec/lib/protobuf/generators/extension_generator_spec.rb +12 -13
- data/spec/lib/protobuf/generators/field_generator_spec.rb +73 -21
- data/spec/lib/protobuf/generators/file_generator_spec.rb +89 -6
- data/spec/lib/protobuf/generators/service_generator_spec.rb +25 -13
- data/spec/lib/protobuf/lifecycle_spec.rb +25 -20
- data/spec/lib/protobuf/message_spec.rb +578 -79
- data/spec/lib/protobuf/optionable_spec.rb +202 -26
- data/spec/lib/protobuf/rpc/client_spec.rb +16 -16
- data/spec/lib/protobuf/rpc/connectors/base_spec.rb +167 -13
- data/spec/lib/protobuf/rpc/connectors/connector_spec.rb +4 -5
- data/spec/lib/protobuf/rpc/connectors/http_spec.rb +13 -11
- data/spec/lib/protobuf/rpc/connectors/ping_spec.rb +69 -0
- data/spec/lib/protobuf/rpc/connectors/socket_spec.rb +6 -7
- data/spec/lib/protobuf/rpc/connectors/zmq_spec.rb +35 -52
- data/spec/lib/protobuf/rpc/middleware/exception_handler_spec.rb +10 -10
- data/spec/lib/protobuf/rpc/middleware/logger_spec.rb +11 -11
- data/spec/lib/protobuf/rpc/middleware/request_decoder_spec.rb +23 -23
- data/spec/lib/protobuf/rpc/middleware/response_encoder_spec.rb +11 -11
- data/spec/lib/protobuf/rpc/middleware/statsd_spec.rb +6 -6
- data/spec/lib/protobuf/rpc/servers/http/server_spec.rb +47 -44
- data/spec/lib/protobuf/rpc/servers/socket_server_spec.rb +6 -6
- data/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb +12 -10
- data/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb +11 -11
- data/spec/lib/protobuf/rpc/servers/zmq/worker_spec.rb +7 -7
- data/spec/lib/protobuf/rpc/service_directory_spec.rb +47 -49
- data/spec/lib/protobuf/rpc/service_dispatcher_spec.rb +8 -25
- data/spec/lib/protobuf/rpc/service_filters_spec.rb +102 -69
- data/spec/lib/protobuf/rpc/service_spec.rb +37 -36
- data/spec/lib/protobuf/rpc/stat_spec.rb +7 -9
- data/spec/lib/protobuf/varint_spec.rb +29 -0
- data/spec/lib/protobuf_spec.rb +55 -28
- data/spec/spec_helper.rb +12 -27
- data/spec/support/all.rb +0 -1
- data/spec/support/packed_field.rb +4 -3
- data/spec/support/{test → protos}/all_types.data.bin +0 -0
- data/spec/support/{test → protos}/all_types.data.txt +0 -0
- data/spec/support/{test → protos}/enum.pb.rb +8 -4
- data/spec/support/{test → protos}/enum.proto +4 -1
- data/spec/support/{test → protos}/extreme_values.data.bin +0 -0
- data/spec/support/protos/google_unittest.bin +0 -0
- data/spec/support/protos/google_unittest.pb.rb +798 -0
- data/spec/support/{test → protos}/google_unittest.proto +237 -66
- data/spec/support/protos/google_unittest_custom_options.bin +0 -0
- data/spec/support/protos/google_unittest_custom_options.pb.rb +268 -0
- data/spec/support/protos/google_unittest_custom_options.proto +424 -0
- data/spec/support/protos/google_unittest_import.pb.rb +55 -0
- data/spec/support/{test → protos}/google_unittest_import.proto +19 -10
- data/spec/support/protos/google_unittest_import_public.pb.rb +31 -0
- data/spec/support/{test → protos}/google_unittest_import_public.proto +8 -5
- data/spec/support/{test → protos}/multi_field_extensions.pb.rb +5 -2
- data/spec/support/{test → protos}/multi_field_extensions.proto +2 -0
- data/spec/support/{test → protos}/resource.pb.rb +47 -11
- data/spec/support/{test → protos}/resource.proto +24 -1
- data/spec/support/resource_service.rb +23 -0
- data/spec/support/server.rb +32 -61
- metadata +119 -59
- data/lib/protobuf/deprecator.rb +0 -42
- data/lib/protobuf/logger.rb +0 -93
- data/lib/protobuf/rpc/connector.rb +0 -21
- data/lib/protobuf/rpc/connectors/common.rb +0 -172
- data/spec/data/data.bin +0 -3
- data/spec/data/types.bin +0 -0
- data/spec/lib/protobuf/logger_spec.rb +0 -145
- data/spec/lib/protobuf/rpc/connector_spec.rb +0 -26
- data/spec/lib/protobuf/rpc/connectors/common_spec.rb +0 -170
- data/spec/support/test/defaults.pb.rb +0 -25
- data/spec/support/test/defaults.proto +0 -9
- data/spec/support/test/extended.pb.rb +0 -22
- data/spec/support/test/extended.proto +0 -10
- data/spec/support/test/google_unittest.pb.rb +0 -543
- data/spec/support/test/google_unittest_import.pb.rb +0 -37
- data/spec/support/test/google_unittest_import_public.pb.rb +0 -8
- data/spec/support/test/resource_service.rb +0 -26
- data/spec/support/tolerance_matcher.rb +0 -40
@@ -1,111 +1,119 @@
|
|
1
|
-
require '
|
1
|
+
require 'set'
|
2
|
+
|
2
3
|
require 'protobuf/rpc/servers/socket/worker'
|
3
4
|
|
4
5
|
module Protobuf
|
5
6
|
module Rpc
|
6
7
|
module Socket
|
7
8
|
class Server
|
8
|
-
include ::Protobuf::
|
9
|
+
include ::Protobuf::Logging
|
9
10
|
|
10
11
|
AUTO_COLLECT_TIMEOUT = 5 # seconds
|
11
12
|
|
13
|
+
private
|
14
|
+
|
15
|
+
attr_accessor :threshold, :host, :port, :backlog
|
16
|
+
attr_writer :running
|
17
|
+
|
18
|
+
public
|
19
|
+
|
20
|
+
attr_reader :running
|
21
|
+
alias :running? running
|
22
|
+
|
12
23
|
def initialize(options)
|
13
|
-
|
24
|
+
self.running = false
|
25
|
+
self.host = options.fetch(:host)
|
26
|
+
self.port = options.fetch(:port)
|
27
|
+
self.backlog = options.fetch(:backlog, 100)
|
28
|
+
self.threshold = options.fetch(:threshold, 100)
|
29
|
+
end
|
30
|
+
|
31
|
+
def threads
|
32
|
+
@threads ||= []
|
33
|
+
end
|
34
|
+
|
35
|
+
def working
|
36
|
+
@working ||= Set.new
|
14
37
|
end
|
15
38
|
|
16
39
|
def cleanup?
|
17
|
-
# every
|
18
|
-
|
40
|
+
# every `threshold` connections run a cleanup routine after closing the response
|
41
|
+
!threads.empty? && threads.size % threshold == 0
|
19
42
|
end
|
20
43
|
|
21
44
|
def cleanup_threads
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
t[:thread].join
|
29
|
-
@working.delete(t[:socket])
|
30
|
-
false
|
45
|
+
logger.debug { sign_message("Thread cleanup - #{threads.size} - start") }
|
46
|
+
|
47
|
+
threads.delete_if do |hash|
|
48
|
+
unless (thread = hash.fetch(:thread)).alive?
|
49
|
+
thread.join
|
50
|
+
working.delete(hash.fetch(:socket))
|
31
51
|
end
|
32
52
|
end
|
33
53
|
|
34
|
-
|
54
|
+
logger.debug { sign_message("Thread cleanup - #{threads.size} - complete") }
|
35
55
|
end
|
36
56
|
|
37
57
|
def log_signature
|
38
|
-
@_log_signature ||= "server-#{self.class.name}"
|
58
|
+
@_log_signature ||= "[server-#{self.class.name}]"
|
39
59
|
end
|
40
60
|
|
41
61
|
def new_worker(socket)
|
42
62
|
Thread.new(socket) do |sock|
|
43
|
-
::Protobuf::Rpc::Socket::Worker.new(sock)
|
44
|
-
s.close
|
45
|
-
end
|
63
|
+
::Protobuf::Rpc::Socket::Worker.new(sock, &:close)
|
46
64
|
end
|
47
65
|
end
|
48
66
|
|
49
67
|
def run
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
cleanup_threads if cleanup?
|
68
|
+
logger.debug { sign_message("Run") }
|
69
|
+
|
70
|
+
server = ::TCPServer.new(host, port)
|
71
|
+
fail "The server was unable to start properly." if server.closed?
|
72
|
+
|
73
|
+
begin
|
74
|
+
server.listen(backlog)
|
75
|
+
listen_fds = [server]
|
76
|
+
self.running = true
|
77
|
+
|
78
|
+
while running?
|
79
|
+
logger.debug { sign_message("Waiting for connections") }
|
80
|
+
ready_cnxns = begin
|
81
|
+
IO.select(listen_fds, [], [], AUTO_COLLECT_TIMEOUT)
|
82
|
+
rescue IOError
|
83
|
+
nil
|
84
|
+
end
|
85
|
+
|
86
|
+
if ready_cnxns
|
87
|
+
ready_cnxns.first.each do |client|
|
88
|
+
case
|
89
|
+
when !running?
|
90
|
+
# no-op
|
91
|
+
when client == server
|
92
|
+
logger.debug { sign_message("Accepted new connection") }
|
93
|
+
client, _sockaddr = server.accept
|
94
|
+
listen_fds << client
|
95
|
+
else
|
96
|
+
unless working.include?(client)
|
97
|
+
working << listen_fds.delete(client)
|
98
|
+
logger.debug { sign_message("Working") }
|
99
|
+
threads << { :thread => new_worker(client), :socket => client }
|
100
|
+
|
101
|
+
cleanup_threads if cleanup?
|
102
|
+
end
|
86
103
|
end
|
87
104
|
end
|
105
|
+
elsif threads.size > 1
|
106
|
+
# Run a cleanup if select times out while waiting
|
107
|
+
cleanup_threads
|
88
108
|
end
|
89
|
-
else
|
90
|
-
# Run a cleanup if select times out while waiting
|
91
|
-
cleanup_threads if @threads.size > 1
|
92
109
|
end
|
110
|
+
ensure
|
111
|
+
server.close
|
93
112
|
end
|
94
|
-
|
95
|
-
rescue Errno::EADDRINUSE
|
96
|
-
raise
|
97
|
-
rescue
|
98
|
-
# Closing the server causes the loop to raise an exception here
|
99
|
-
raise #if running?
|
100
|
-
end
|
101
|
-
|
102
|
-
def running?
|
103
|
-
!!@running
|
104
113
|
end
|
105
114
|
|
106
115
|
def stop
|
107
|
-
|
108
|
-
@server.try(:close)
|
116
|
+
self.running = false
|
109
117
|
end
|
110
118
|
end
|
111
119
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
require 'protobuf/rpc/server'
|
2
|
-
require 'protobuf/
|
2
|
+
require 'protobuf/logging'
|
3
3
|
|
4
4
|
module Protobuf
|
5
5
|
module Rpc
|
6
6
|
module Socket
|
7
7
|
class Worker
|
8
8
|
include ::Protobuf::Rpc::Server
|
9
|
-
include ::Protobuf::
|
9
|
+
include ::Protobuf::Logging
|
10
10
|
|
11
11
|
def initialize(sock, &complete_cb)
|
12
12
|
@socket = sock
|
@@ -33,7 +33,7 @@ module Protobuf
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def send_data(data)
|
36
|
-
|
36
|
+
fail 'Socket closed unexpectedly' unless socket_writable?
|
37
37
|
response_buffer = Protobuf::Rpc::Buffer.new(:write)
|
38
38
|
response_buffer.set_data(data)
|
39
39
|
|
@@ -44,7 +44,7 @@ module Protobuf
|
|
44
44
|
end
|
45
45
|
|
46
46
|
def log_signature
|
47
|
-
@_log_signature ||= "server-#{self.class}-#{object_id}"
|
47
|
+
@_log_signature ||= "[server-#{self.class}-#{object_id}]"
|
48
48
|
end
|
49
49
|
|
50
50
|
def socket_writable?
|
@@ -2,33 +2,45 @@ module Protobuf
|
|
2
2
|
module Rpc
|
3
3
|
class SocketRunner
|
4
4
|
|
5
|
+
private
|
6
|
+
|
7
|
+
attr_accessor :server
|
8
|
+
|
9
|
+
public
|
10
|
+
|
5
11
|
def initialize(options)
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
@server = ::Protobuf::Rpc::Socket::Server.new(@options)
|
12
|
+
options = case
|
13
|
+
when options.is_a?(OpenStruct) then
|
14
|
+
options.marshal_dump
|
15
|
+
when options.respond_to?(:to_hash) then
|
16
|
+
options.to_hash.symbolize_keys
|
17
|
+
else
|
18
|
+
fail "Cannot parser Socket Server - server options"
|
19
|
+
end
|
20
|
+
|
21
|
+
self.server = ::Protobuf::Rpc::Socket::Server.new(options)
|
18
22
|
end
|
19
23
|
|
20
24
|
def run
|
21
25
|
yield if block_given?
|
22
|
-
|
26
|
+
server.run
|
23
27
|
end
|
24
28
|
|
25
29
|
def running?
|
26
|
-
|
30
|
+
server.running?
|
27
31
|
end
|
28
32
|
|
29
33
|
def stop
|
30
|
-
|
34
|
+
server.stop
|
31
35
|
end
|
32
36
|
end
|
33
37
|
end
|
34
38
|
end
|
39
|
+
|
40
|
+
module Protobuf
|
41
|
+
module Rpc
|
42
|
+
module Servers # bad file namespacing
|
43
|
+
SocketRunner = ::Protobuf::Rpc::SocketRunner
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -23,39 +23,69 @@ module Protobuf
|
|
23
23
|
|
24
24
|
def run
|
25
25
|
@idle_workers = []
|
26
|
+
@running = true
|
26
27
|
|
27
28
|
loop do
|
28
|
-
|
29
|
-
|
30
|
-
end
|
31
|
-
|
32
|
-
rc = @poller.poll(500)
|
29
|
+
process_local_queue
|
30
|
+
rc = @poller.poll(broker_polling_milliseconds)
|
33
31
|
|
34
32
|
# The server was shutdown and no requests are pending
|
35
|
-
break if rc == 0 && !running?
|
36
|
-
|
33
|
+
break if rc == 0 && !running? && @server.workers.empty?
|
37
34
|
# Something went wrong
|
38
35
|
break if rc == -1
|
39
36
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
process_frontend
|
44
|
-
when @backend_socket
|
45
|
-
process_backend
|
46
|
-
end
|
47
|
-
end
|
37
|
+
check_and_process_backend
|
38
|
+
process_local_queue # Fair ordering so queued requests get in before new requests
|
39
|
+
check_and_process_frontend
|
48
40
|
end
|
49
41
|
ensure
|
50
42
|
teardown
|
43
|
+
@running = false
|
51
44
|
end
|
52
45
|
|
53
46
|
def running?
|
54
|
-
@
|
47
|
+
@running && @server.running?
|
55
48
|
end
|
56
49
|
|
57
50
|
private
|
58
51
|
|
52
|
+
def backend_poll_weight
|
53
|
+
@backend_poll_weight ||= [ENV["PB_ZMQ_SERVER_BACKEND_POLL_WEIGHT"].to_i, 1].max
|
54
|
+
end
|
55
|
+
|
56
|
+
def broker_polling_milliseconds
|
57
|
+
@broker_polling_milliseconds ||= [ENV["PB_ZMQ_BROKER_POLLING_MILLISECONDS"].to_i, 500].max
|
58
|
+
end
|
59
|
+
|
60
|
+
def check_and_process_backend
|
61
|
+
readables_include_backend = @poller.readables.include?(@backend_socket)
|
62
|
+
message_count_read_from_backend = 0
|
63
|
+
|
64
|
+
while readables_include_backend && message_count_read_from_backend < backend_poll_weight
|
65
|
+
message_count_read_from_backend += 1
|
66
|
+
process_backend
|
67
|
+
@poller.poll_nonblock
|
68
|
+
readables_include_backend = @poller.readables.include?(@backend_socket)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def check_and_process_frontend
|
73
|
+
readables_include_frontend = @poller.readables.include?(@frontend_socket)
|
74
|
+
message_count_read_from_frontend = 0
|
75
|
+
|
76
|
+
while readables_include_frontend && message_count_read_from_frontend < frontend_poll_weight
|
77
|
+
message_count_read_from_frontend += 1
|
78
|
+
process_frontend
|
79
|
+
break unless local_queue_available? # no need to read frontend just to throw away messages, will prioritize backend when full
|
80
|
+
@poller.poll_nonblock
|
81
|
+
readables_include_frontend = @poller.readables.include?(@frontend_socket)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def frontend_poll_weight
|
86
|
+
@frontend_poll_weight ||= [ENV["PB_ZMQ_SERVER_FRONTEND_POLL_WEIGHT"].to_i, 1].max
|
87
|
+
end
|
88
|
+
|
59
89
|
def init_backend_socket
|
60
90
|
@backend_socket = @zmq_context.socket(ZMQ::ROUTER)
|
61
91
|
zmq_error_check(@backend_socket.bind(@server.backend_uri))
|
@@ -67,7 +97,7 @@ module Protobuf
|
|
67
97
|
end
|
68
98
|
|
69
99
|
def init_local_queue
|
70
|
-
@local_queue =
|
100
|
+
@local_queue = []
|
71
101
|
end
|
72
102
|
|
73
103
|
def init_poller
|
@@ -77,19 +107,28 @@ module Protobuf
|
|
77
107
|
end
|
78
108
|
|
79
109
|
def init_zmq_context
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
110
|
+
@zmq_context =
|
111
|
+
if inproc?
|
112
|
+
@server.zmq_context
|
113
|
+
else
|
114
|
+
ZMQ::Context.new
|
115
|
+
end
|
85
116
|
end
|
86
117
|
|
87
118
|
def inproc?
|
88
119
|
!!@server.try(:inproc?)
|
89
120
|
end
|
90
121
|
|
122
|
+
def local_queue_available?
|
123
|
+
local_queue.size < local_queue_max_size && running?
|
124
|
+
end
|
125
|
+
|
126
|
+
def local_queue_max_size
|
127
|
+
@local_queue_max_size ||= [ENV["PB_ZMQ_SERVER_QUEUE_MAX_SIZE"].to_i, 5].max
|
128
|
+
end
|
129
|
+
|
91
130
|
def process_backend
|
92
|
-
worker,
|
131
|
+
worker, _ignore, *frames = read_from_backend
|
93
132
|
|
94
133
|
@idle_workers << worker
|
95
134
|
|
@@ -102,16 +141,16 @@ module Protobuf
|
|
102
141
|
address, _, message, *frames = read_from_frontend
|
103
142
|
|
104
143
|
if message == ::Protobuf::Rpc::Zmq::CHECK_AVAILABLE_MESSAGE
|
105
|
-
if
|
106
|
-
write_to_frontend([address,
|
144
|
+
if local_queue_available?
|
145
|
+
write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::WORKERS_AVAILABLE])
|
107
146
|
else
|
108
|
-
write_to_frontend([address,
|
147
|
+
write_to_frontend([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, ::Protobuf::Rpc::Zmq::NO_WORKERS_AVAILABLE])
|
109
148
|
end
|
110
149
|
else
|
111
|
-
if @idle_workers.
|
112
|
-
|
150
|
+
if @idle_workers.empty? # rubocop:disable Style/IfInsideElse
|
151
|
+
local_queue << [address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message].concat(frames)
|
113
152
|
else
|
114
|
-
|
153
|
+
write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat([address, ::Protobuf::Rpc::Zmq::EMPTY_STRING, message]).concat(frames))
|
115
154
|
end
|
116
155
|
end
|
117
156
|
end
|
@@ -120,7 +159,7 @@ module Protobuf
|
|
120
159
|
return if local_queue.empty?
|
121
160
|
return if @idle_workers.empty?
|
122
161
|
|
123
|
-
write_to_backend([@idle_workers.shift,
|
162
|
+
write_to_backend([@idle_workers.shift, ::Protobuf::Rpc::Zmq::EMPTY_STRING].concat(local_queue.shift))
|
124
163
|
process_local_queue
|
125
164
|
end
|
126
165
|
|
@@ -139,7 +178,7 @@ module Protobuf
|
|
139
178
|
def teardown
|
140
179
|
@frontend_socket.try(:close)
|
141
180
|
@backend_socket.try(:close)
|
142
|
-
@zmq_context.try(:terminate)
|
181
|
+
@zmq_context.try(:terminate) unless inproc?
|
143
182
|
end
|
144
183
|
|
145
184
|
def write_to_backend(frames)
|