protobuf-nats 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.travis.yml +8 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +118 -0
- data/Rakefile +6 -0
- data/bench/client.rb +26 -0
- data/bench/real_client.rb +21 -0
- data/bench/real_server.sh +1 -0
- data/bench/server.rb +39 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/examples/warehouse/app.rb +55 -0
- data/examples/warehouse/start_client.sh +7 -0
- data/examples/warehouse/start_server.sh +1 -0
- data/lib/protobuf/nats/client.rb +99 -0
- data/lib/protobuf/nats/config.rb +69 -0
- data/lib/protobuf/nats/fixed_thread_pool_with_no_queue.rb +32 -0
- data/lib/protobuf/nats/runner.rb +55 -0
- data/lib/protobuf/nats/server.rb +106 -0
- data/lib/protobuf/nats/version.rb +5 -0
- data/lib/protobuf/nats.rb +57 -0
- data/protobuf-nats.gemspec +43 -0
- metadata +194 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8941cd6d0acf196cafb26b994e7cb71156cc9ddf
|
4
|
+
data.tar.gz: f87f6f7c86874688d94785e177e54f35950d91c7
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: abdcc5311c25b5598a8bd73ec344ab3c12a8072f689a7473645e21e6ba61e477ede2e96deea566a4a15a0f3da0d8b27a3036ce062c497d487bbb7999158d5dd8
|
7
|
+
data.tar.gz: f3dda5036de14aae93b0e07933897f835ad944c0e0798f9dc1b6a135bba83d7f82c2a8a265b45bf7c393dac013863519e89bc39315420c30269f8744459055b8
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 Brandon Dewitt
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
Protobuf::Nats
|
2
|
+
==============
|
3
|
+
|
4
|
+
An rpc client and server library built using the `protobuf` gem and the NATS protocol.
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
Add this line to your application's Gemfile:
|
9
|
+
|
10
|
+
```ruby
|
11
|
+
gem 'protobuf-nats'
|
12
|
+
```
|
13
|
+
|
14
|
+
And then execute:
|
15
|
+
|
16
|
+
$ bundle
|
17
|
+
|
18
|
+
Or install it yourself as:
|
19
|
+
|
20
|
+
$ gem install protobuf-nats
|
21
|
+
|
22
|
+
## Configuring
|
23
|
+
|
24
|
+
The client and server are configured via environment variables defined in the `pure-ruby-nats` gem. However, there are a
|
25
|
+
few params which cannot be set: `servers`, `uses_tls` and `connect_timeout`, so those my be defined in a yml file.
|
26
|
+
|
27
|
+
The library will automatically look for a file with a relative path of `config/protobuf_nats.yml`, but you may override
|
28
|
+
this by specifying a different file via the `PROTOBUF_NATS_CONFIG_PATH` env variable.
|
29
|
+
|
30
|
+
An example config looks like this:
|
31
|
+
```
|
32
|
+
# Stored at config/protobuf_nats.yml
|
33
|
+
---
|
34
|
+
production:
|
35
|
+
servers:
|
36
|
+
- "nats://127.0.0.1:4222"
|
37
|
+
- "nats://127.0.0.1:4223"
|
38
|
+
- "nats://127.0.0.1:4224"
|
39
|
+
uses_tls: true
|
40
|
+
connect_timeout: 2
|
41
|
+
```
|
42
|
+
|
43
|
+
## Usage
|
44
|
+
|
45
|
+
This library is designed to be an alternative transport implementation used by the `protobuf` gem. In order to make
|
46
|
+
`protobuf` use this library, you need to set the following env variable:
|
47
|
+
|
48
|
+
```
|
49
|
+
PB_SERVER_TYPE="protobuf/nats/runner"
|
50
|
+
PB_CLIENT_TYPE="protobuf/nats/client"
|
51
|
+
```
|
52
|
+
|
53
|
+
## Example
|
54
|
+
|
55
|
+
NOTE: For a more detailed example, look at the `warehouse` app in the `examples` directory of this project.
|
56
|
+
|
57
|
+
Here's a tl;dr example. You might have a protobuf definition and implementation like this:
|
58
|
+
|
59
|
+
```ruby
|
60
|
+
require "protobuf/nats"
|
61
|
+
|
62
|
+
class User < ::Protobuf::Message
|
63
|
+
optional :int64, :id, 1
|
64
|
+
optional :string, :username, 2
|
65
|
+
end
|
66
|
+
|
67
|
+
class UserService < ::Protobuf::Rpc::Service
|
68
|
+
rpc :create, User, User
|
69
|
+
|
70
|
+
def create
|
71
|
+
respond_with User.new(:id => 123, :username => request.username)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
```
|
75
|
+
|
76
|
+
Let's assume we saved this in a file called `app.rb`
|
77
|
+
|
78
|
+
We can now start an rpc server using the protobuf-nats runner and client:
|
79
|
+
|
80
|
+
```
|
81
|
+
$ export PB_SERVER_TYPE="protobuf/nats/runner"
|
82
|
+
$ export PB_CLIENT_TYPE="protobuf/nats/client"
|
83
|
+
$ bundle exec rpc_server start ./app.rb
|
84
|
+
...
|
85
|
+
I, [2017-03-24T12:16:02.539930 #12512] INFO -- : Creating subscriptions:
|
86
|
+
I, [2017-03-24T12:16:02.543927 #12512] INFO -- : - rpc.user_service.create
|
87
|
+
...
|
88
|
+
```
|
89
|
+
|
90
|
+
And we can start a client and begin communicating:
|
91
|
+
|
92
|
+
```
|
93
|
+
$ export PB_SERVER_TYPE="protobuf/nats/runner"
|
94
|
+
$ export PB_CLIENT_TYPE="protobuf/nats/client"
|
95
|
+
$ bundle exec irb -r ./app
|
96
|
+
irb(main):001:0> UserService.client.create(User.new(:username => "testing 123"))
|
97
|
+
=> #<User id=123 username="testing 123">
|
98
|
+
```
|
99
|
+
|
100
|
+
And we can see the message was sent to the server and the server replied with a user which now has an `id`.
|
101
|
+
|
102
|
+
If we were to add another service endpoint called `search` to the `UserService` but fail to define an instance method
|
103
|
+
`search`, then `protobuf-nats` will not subscribe to that route.
|
104
|
+
|
105
|
+
## Development
|
106
|
+
|
107
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
108
|
+
|
109
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
110
|
+
|
111
|
+
## Contributing
|
112
|
+
|
113
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/abrandoned/protobuf-nats.
|
114
|
+
|
115
|
+
|
116
|
+
## License
|
117
|
+
|
118
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bench/client.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
ENV["PB_CLIENT_TYPE"] = "protobuf/nats/client"
|
2
|
+
ENV["PB_SERVER_TYPE"] = "protobuf/nats/runner"
|
3
|
+
|
4
|
+
require "benchmark/ips"
|
5
|
+
require "./examples/warehouse/app"
|
6
|
+
|
7
|
+
payload = ::Warehouse::Shipment.new(:guid => ::SecureRandom.uuid, :address => "123 yolo st")
|
8
|
+
request = ::Protobuf::Socketrpc::Request.new(:service_name => "::Warehouse::ShipmentService",
|
9
|
+
:method_name => "create",
|
10
|
+
:request_proto => payload.encode).encode
|
11
|
+
|
12
|
+
nats = ::NATS::IO::Client.new
|
13
|
+
nats.connect({:servers => ["nats://127.0.0.1:4222"]})
|
14
|
+
|
15
|
+
subscription_key_and_queue = "rpc.warehouse.shipment_service.create"
|
16
|
+
|
17
|
+
Protobuf::Logging.logger = ::Logger.new(nil)
|
18
|
+
|
19
|
+
Benchmark.ips do |config|
|
20
|
+
config.warmup = 10
|
21
|
+
config.time = 10
|
22
|
+
|
23
|
+
config.report("single threaded performance") do
|
24
|
+
nats.request(subscription_key_and_queue, request)
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
ENV["PB_CLIENT_TYPE"] = "protobuf/nats/client"
|
2
|
+
ENV["PB_SERVER_TYPE"] = "protobuf/nats/runner"
|
3
|
+
|
4
|
+
require "benchmark/ips"
|
5
|
+
require "./examples/warehouse/app"
|
6
|
+
|
7
|
+
payload = ::Warehouse::Shipment.new(:guid => ::SecureRandom.uuid, :address => "123 yolo st")
|
8
|
+
nats = ::NATS::IO::Client.new
|
9
|
+
nats.connect({:servers => ["nats://127.0.0.1:4222"]})
|
10
|
+
|
11
|
+
Protobuf::Logging.logger = ::Logger.new(nil)
|
12
|
+
|
13
|
+
Benchmark.ips do |config|
|
14
|
+
config.warmup = 10
|
15
|
+
config.time = 10
|
16
|
+
|
17
|
+
config.report("single threaded performance") do
|
18
|
+
req = Warehouse::Shipment.new(:guid => SecureRandom.uuid)
|
19
|
+
Warehouse::ShipmentService.client.create(req)
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
PB_SERVER_TYPE="protobuf/nats/runner" PB_CLIENT_TYPE="protobuf/nats/client" bundle exec rpc_server start --threads=20 ./examples/warehouse/app.rb > /dev/null
|
data/bench/server.rb
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
require "nats/io/client"
|
2
|
+
require "concurrent"
|
3
|
+
require "protobuf"
|
4
|
+
require 'protobuf/message'
|
5
|
+
require 'protobuf/rpc/service'
|
6
|
+
|
7
|
+
module Warehouse
|
8
|
+
class Shipment < ::Protobuf::Message
|
9
|
+
optional :string, :guid, 1
|
10
|
+
optional :string, :address, 2
|
11
|
+
optional :double, :price, 3
|
12
|
+
optional :string, :package_guid, 4
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
$thread_pool = ::Concurrent::FixedThreadPool.new(20)
|
17
|
+
payload = ::Warehouse::Shipment.new(:guid => ::SecureRandom.uuid, :address => "123 yolo st")
|
18
|
+
$response = ::Protobuf::Socketrpc::Response.new(:response_proto => payload.encode).encode
|
19
|
+
|
20
|
+
$nats = ::NATS::IO::Client.new
|
21
|
+
$nats.connect({:servers => ["nats://127.0.0.1:4222"]})
|
22
|
+
|
23
|
+
subscription_key_and_queue = "Warehouse::ShipmentService::create"
|
24
|
+
$nats.subscribe(subscription_key_and_queue, :queue => subscription_key_and_queue) do |request_data, reply_id, _subject|
|
25
|
+
::Concurrent::Promise.new(:executor => $thread_pool).then do
|
26
|
+
$nats.publish(reply_id, $response)
|
27
|
+
end.on_error do |error|
|
28
|
+
logger.error error
|
29
|
+
if error.respond_to?(:backtrace) && error.backtrace.is_a?(::Array)
|
30
|
+
logger.error error.backtrace.join("\n")
|
31
|
+
end
|
32
|
+
end.execute
|
33
|
+
end
|
34
|
+
|
35
|
+
puts "Server started!"
|
36
|
+
|
37
|
+
loop do
|
38
|
+
sleep 1
|
39
|
+
end
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "protobuf/nats"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
require "protobuf/nats"
|
2
|
+
|
3
|
+
require 'protobuf/message'
|
4
|
+
require 'protobuf/rpc/service'
|
5
|
+
|
6
|
+
module Warehouse
|
7
|
+
|
8
|
+
##
|
9
|
+
# Message Classes
|
10
|
+
#
|
11
|
+
class Shipment < ::Protobuf::Message; end
|
12
|
+
class ShipmentRequest < ::Protobuf::Message; end
|
13
|
+
class Shipments < ::Protobuf::Message; end
|
14
|
+
|
15
|
+
|
16
|
+
##
|
17
|
+
# Message Fields
|
18
|
+
#
|
19
|
+
class Shipment
|
20
|
+
optional :string, :guid, 1
|
21
|
+
optional :string, :address, 2
|
22
|
+
optional :double, :price, 3
|
23
|
+
optional :string, :package_guid, 4
|
24
|
+
end
|
25
|
+
|
26
|
+
class ShipmentRequest
|
27
|
+
repeated :string, :guid, 1
|
28
|
+
repeated :string, :address, 2
|
29
|
+
repeated :string, :package_guid, 3
|
30
|
+
end
|
31
|
+
|
32
|
+
class Shipments
|
33
|
+
repeated ::Warehouse::Shipment, :records, 1
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
##
|
38
|
+
# Service Classes
|
39
|
+
#
|
40
|
+
class ShipmentService < ::Protobuf::Rpc::Service
|
41
|
+
rpc :create, ::Warehouse::Shipment, ::Warehouse::Shipment
|
42
|
+
rpc :not_implemented, ::Warehouse::Shipment, ::Warehouse::Shipment
|
43
|
+
rpc :search, ::Warehouse::ShipmentRequest, ::Warehouse::Shipments
|
44
|
+
|
45
|
+
def create
|
46
|
+
respond_with request
|
47
|
+
end
|
48
|
+
|
49
|
+
def search
|
50
|
+
shipment = ::Warehouse::Shipment.new(:guid => SecureRandom.uuid, :address => "123 LAME ST", :price => 100.0, :package_guid => SecureRandom.uuid)
|
51
|
+
respond_with ::Warehouse::Shipments.new(:records => [shipment])
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
echo "Examples:"
|
2
|
+
echo "req = Warehouse::ShipmentRequest.new"
|
3
|
+
echo "res = Warehouse::ShipmentService.client.search(req)"
|
4
|
+
echo ""
|
5
|
+
echo "req = Warehouse::Shipment.new"
|
6
|
+
echo "res = Warehouse::ShipmentService.client.create(req)"
|
7
|
+
PB_SERVER_TYPE="protobuf/nats/runner" PB_CLIENT_TYPE="protobuf/nats/client" bundle exec irb -I lib -r ./examples/warehouse/app.rb
|
@@ -0,0 +1 @@
|
|
1
|
+
PB_SERVER_TYPE="protobuf/nats/runner" PB_CLIENT_TYPE="protobuf/nats/client" bundle exec rpc_server start ./examples/warehouse/app.rb
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require "protobuf/nats"
|
2
|
+
require "protobuf/rpc/connectors/base"
|
3
|
+
require "monitor"
|
4
|
+
|
5
|
+
module Protobuf
|
6
|
+
module Nats
|
7
|
+
class Client < ::Protobuf::Rpc::Connectors::Base
|
8
|
+
def initialize(options)
|
9
|
+
# may need to override to setup connection at this stage ... may also do on load of class
|
10
|
+
super
|
11
|
+
|
12
|
+
::Protobuf::Nats.ensure_client_nats_connection_was_started
|
13
|
+
end
|
14
|
+
|
15
|
+
def close_connection
|
16
|
+
# no-op (I think for now), the connection to server is persistent
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.subscription_key_cache
|
20
|
+
@subscription_key_cache ||= {}
|
21
|
+
end
|
22
|
+
|
23
|
+
def send_request
|
24
|
+
retries ||= 3
|
25
|
+
|
26
|
+
setup_connection
|
27
|
+
request_options = {:timeout => 60, :ack_timeout => 5}
|
28
|
+
@response_data = nats_request_with_two_responses(cached_subscription_key, @request_data, request_options)
|
29
|
+
parse_response
|
30
|
+
rescue ::NATS::IO::Timeout
|
31
|
+
# Nats response timeout.
|
32
|
+
retry if (retries -= 1) > 0
|
33
|
+
raise
|
34
|
+
end
|
35
|
+
|
36
|
+
def cached_subscription_key
|
37
|
+
klass = @options[:service]
|
38
|
+
method_name = @options[:method]
|
39
|
+
|
40
|
+
method_name_cache = self.class.subscription_key_cache[klass] ||= {}
|
41
|
+
method_name_cache[method_name] ||= begin
|
42
|
+
::Protobuf::Nats.subscription_key(klass, method_name)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# This is a request that expects two responses.
|
47
|
+
# 1. An ACK from the server. We use a shorter timeout.
|
48
|
+
# 2. A PB message from the server. We use a longer timoeut.
|
49
|
+
def nats_request_with_two_responses(subject, data, opts)
|
50
|
+
nats = Protobuf::Nats.client_nats_connection
|
51
|
+
inbox = nats.new_inbox
|
52
|
+
lock = ::Monitor.new
|
53
|
+
ack_condition = lock.new_cond
|
54
|
+
pb_response_condition = lock.new_cond
|
55
|
+
response = nil
|
56
|
+
sid = nats.subscribe(inbox, :max => 2) do |message|
|
57
|
+
lock.synchronize do
|
58
|
+
case message
|
59
|
+
when ::Protobuf::Nats::Messages::ACK
|
60
|
+
ack_condition.signal
|
61
|
+
next
|
62
|
+
else
|
63
|
+
response = message
|
64
|
+
pb_response_condition.signal
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
lock.synchronize do
|
70
|
+
# Publish to server
|
71
|
+
nats.publish(subject, data, inbox)
|
72
|
+
|
73
|
+
# Wait for the ACK from the server
|
74
|
+
ack_timeout = opts[:ack_timeout] || 5
|
75
|
+
with_timeout(ack_timeout) { ack_condition.wait(ack_timeout) }
|
76
|
+
|
77
|
+
# Wait for the protobuf response
|
78
|
+
timeout = opts[:timeout] || 60
|
79
|
+
with_timeout(timeout) { pb_response_condition.wait(timeout) } unless response
|
80
|
+
end
|
81
|
+
|
82
|
+
response
|
83
|
+
ensure
|
84
|
+
# Ensure we don't leave a subscription sitting around.
|
85
|
+
nats.unsubscribe(sid) if response.nil?
|
86
|
+
end
|
87
|
+
|
88
|
+
# This is a copy of #with_nats_timeout
|
89
|
+
def with_timeout(timeout)
|
90
|
+
start_time = ::NATS::MonotonicTime.now
|
91
|
+
yield
|
92
|
+
end_time = ::NATS::MonotonicTime.now
|
93
|
+
duration = end_time - start_time
|
94
|
+
raise ::NATS::IO::Timeout.new("nats: timeout") if duration > timeout
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require "openssl"
|
2
|
+
require "yaml"
|
3
|
+
|
4
|
+
module Protobuf
|
5
|
+
module Nats
|
6
|
+
class Config
|
7
|
+
attr_accessor :uses_tls, :servers, :connect_timeout
|
8
|
+
|
9
|
+
CONFIG_MUTEX = ::Mutex.new
|
10
|
+
|
11
|
+
DEFAULTS = {
|
12
|
+
:uses_tls => false,
|
13
|
+
:connect_timeout => nil,
|
14
|
+
:servers => nil
|
15
|
+
}.freeze
|
16
|
+
|
17
|
+
def initialize
|
18
|
+
DEFAULTS.each_pair do |key, value|
|
19
|
+
__send__("#{key}=", value)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def load_from_yml(reload = false)
|
24
|
+
CONFIG_MUTEX.synchronize do
|
25
|
+
@load_from_yml = nil if reload
|
26
|
+
@load_from_yml ||= begin
|
27
|
+
env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || ENV["APP_ENV"] || "development"
|
28
|
+
|
29
|
+
yaml_config = {}
|
30
|
+
config_path = ENV["PROTOBUF_NATS_CONFIG_PATH"] || ::File.join("config", "protobuf_nats.yml")
|
31
|
+
absolute_config_path = ::File.expand_path(config_path)
|
32
|
+
if ::File.exists?(absolute_config_path)
|
33
|
+
yaml_config = ::YAML.load_file(absolute_config_path)[env]
|
34
|
+
end
|
35
|
+
|
36
|
+
DEFAULTS.each_pair do |key, value|
|
37
|
+
setting = yaml_config[key.to_s]
|
38
|
+
__send__("#{key}=", setting) if setting
|
39
|
+
end
|
40
|
+
|
41
|
+
# Reload the connection options hash
|
42
|
+
connection_options(true)
|
43
|
+
|
44
|
+
true
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def connection_options(reload = false)
|
50
|
+
@connection_options = false if reload
|
51
|
+
@connection_options || begin
|
52
|
+
options = {
|
53
|
+
servers: servers,
|
54
|
+
connect_timeout: connect_timeout
|
55
|
+
}
|
56
|
+
options[:tls] = {:context => new_tls_context} if uses_tls
|
57
|
+
options
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def new_tls_context
|
62
|
+
tls_context = ::OpenSSL::SSL::SSLContext.new
|
63
|
+
tls_context.ssl_version = :TLSv1_2
|
64
|
+
tls_context
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require "concurrent"
|
2
|
+
|
3
|
+
module Protobuf
|
4
|
+
module Nats
|
5
|
+
class FixedThreadPoolWithNoQueue < ::Concurrent::FixedThreadPool
|
6
|
+
def ns_initialize(opts)
|
7
|
+
min_length = opts.fetch(:min_threads, DEFAULT_MIN_POOL_SIZE).to_i
|
8
|
+
max_length = opts.fetch(:max_threads, DEFAULT_MAX_POOL_SIZE).to_i
|
9
|
+
idletime = opts.fetch(:idletime, DEFAULT_THREAD_IDLETIMEOUT).to_i
|
10
|
+
@fallback_policy = opts.fetch(:fallback_policy, :abort)
|
11
|
+
|
12
|
+
if ::Concurrent.on_jruby?
|
13
|
+
super(opts)
|
14
|
+
|
15
|
+
# We need to use a synchronous queue to ensure we only perform work
|
16
|
+
# when a thread is available.
|
17
|
+
queue = java.util.concurrent.SynchronousQueue.new
|
18
|
+
@executor = java.util.concurrent.ThreadPoolExecutor.new(
|
19
|
+
min_length, max_length,
|
20
|
+
idletime, java.util.concurrent.TimeUnit::SECONDS,
|
21
|
+
queue, FALLBACK_POLICY_CLASSES[@fallback_policy].new)
|
22
|
+
else
|
23
|
+
# MRI will take a max queue of -1 to mean no queue.
|
24
|
+
opts[:max_queue] = -1
|
25
|
+
super(opts)
|
26
|
+
end
|
27
|
+
|
28
|
+
@max_queue = -1
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require "ostruct"
|
2
|
+
require "thread"
|
3
|
+
|
4
|
+
require "protobuf/nats"
|
5
|
+
|
6
|
+
module Protobuf
|
7
|
+
module Nats
|
8
|
+
class Runner
|
9
|
+
|
10
|
+
def initialize(options)
|
11
|
+
@options = case
|
12
|
+
when options.is_a?(OpenStruct) then
|
13
|
+
options.marshal_dump
|
14
|
+
when options.respond_to?(:to_hash) then
|
15
|
+
options.to_hash.symbolize_keys
|
16
|
+
else
|
17
|
+
fail "Cannot parse Nats Server - server options"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def run
|
22
|
+
@server = ::Protobuf::Nats::Server.new(@options)
|
23
|
+
register_signals
|
24
|
+
@server.run do
|
25
|
+
yield if block_given?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def running?
|
30
|
+
@server.try :running?
|
31
|
+
end
|
32
|
+
|
33
|
+
def stop
|
34
|
+
@server.try :stop
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def register_signals
|
40
|
+
trap(:TRAP) do
|
41
|
+
::Thread.list.each do |thread|
|
42
|
+
logger.info do
|
43
|
+
<<-THREAD_TRACE
|
44
|
+
#{thread.inspect}:
|
45
|
+
#{thread.backtrace.try(:join, $INPUT_RECORD_SEPARATOR)}"
|
46
|
+
THREAD_TRACE
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require "active_support/core_ext/class/subclasses"
|
2
|
+
require "protobuf/nats/fixed_thread_pool_with_no_queue"
|
3
|
+
require "protobuf/rpc/server"
|
4
|
+
require "protobuf/rpc/service"
|
5
|
+
|
6
|
+
module Protobuf
|
7
|
+
module Nats
|
8
|
+
class Server
|
9
|
+
include ::Protobuf::Rpc::Server
|
10
|
+
include ::Protobuf::Logging
|
11
|
+
|
12
|
+
attr_reader :nats, :thread_pool, :subscriptions
|
13
|
+
|
14
|
+
def initialize(options)
|
15
|
+
@options = options
|
16
|
+
@running = true
|
17
|
+
@stopped = false
|
18
|
+
|
19
|
+
@nats = options[:client] || ::NATS::IO::Client.new
|
20
|
+
@nats.connect(::Protobuf::Nats.config.connection_options)
|
21
|
+
|
22
|
+
@thread_pool = ::Protobuf::Nats::FixedThreadPoolWithNoQueue.new(options[:threads])
|
23
|
+
|
24
|
+
@subscriptions = []
|
25
|
+
end
|
26
|
+
|
27
|
+
def service_klasses
|
28
|
+
::Protobuf::Rpc::Service.implemented_services.map(&:safe_constantize)
|
29
|
+
end
|
30
|
+
|
31
|
+
def execute_request_promise(request_data, reply_id)
|
32
|
+
::Concurrent::Promise.new(:executor => thread_pool).then do
|
33
|
+
# Publish an ACK.
|
34
|
+
nats.publish(reply_id, ::Protobuf::Nats::Messages::ACK)
|
35
|
+
# Process request.
|
36
|
+
response_data = handle_request(request_data)
|
37
|
+
# Publish response.
|
38
|
+
nats.publish(reply_id, response_data)
|
39
|
+
end.on_error do |error|
|
40
|
+
logger.error error.to_s
|
41
|
+
if error.respond_to?(:backtrace) && error.backtrace.is_a?(::Array)
|
42
|
+
logger.error error.backtrace.join("\n")
|
43
|
+
end
|
44
|
+
end.execute
|
45
|
+
rescue ::Concurrent::RejectedExecutionError
|
46
|
+
nil
|
47
|
+
end
|
48
|
+
|
49
|
+
def subscribe_to_services
|
50
|
+
logger.info "Creating subscriptions:"
|
51
|
+
|
52
|
+
service_klasses.each do |service_klass|
|
53
|
+
service_klass.rpcs.each do |service_method, _|
|
54
|
+
# Skip services that are not implemented.
|
55
|
+
next unless service_klass.method_defined? service_method
|
56
|
+
|
57
|
+
subscription_key_and_queue = ::Protobuf::Nats.subscription_key(service_klass, service_method)
|
58
|
+
logger.info " - #{subscription_key_and_queue}"
|
59
|
+
|
60
|
+
subscriptions << nats.subscribe(subscription_key_and_queue, :queue => subscription_key_and_queue) do |request_data, reply_id, _subject|
|
61
|
+
unless execute_request_promise(request_data, reply_id)
|
62
|
+
logger.error { "Thread pool is full! Dropping message for: #{subscription_key_and_queue}" }
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def run
|
70
|
+
nats.on_reconnect do
|
71
|
+
logger.warn "Reconnected to NATS server!"
|
72
|
+
end
|
73
|
+
|
74
|
+
nats.on_disconnect do
|
75
|
+
logger.warn "Disconnected from NATS server!"
|
76
|
+
end
|
77
|
+
|
78
|
+
subscribe_to_services
|
79
|
+
|
80
|
+
yield if block_given?
|
81
|
+
|
82
|
+
loop do
|
83
|
+
break unless @running
|
84
|
+
sleep 1
|
85
|
+
end
|
86
|
+
|
87
|
+
subscriptions.each do |subscription_id|
|
88
|
+
nats.unsubscribe(subscription_id)
|
89
|
+
end
|
90
|
+
|
91
|
+
thread_pool.shutdown
|
92
|
+
thread_pool.wait_for_termination
|
93
|
+
ensure
|
94
|
+
@stopped = true
|
95
|
+
end
|
96
|
+
|
97
|
+
def running?
|
98
|
+
@stopped
|
99
|
+
end
|
100
|
+
|
101
|
+
def stop
|
102
|
+
@running = false
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "protobuf/nats/version"
|
2
|
+
|
3
|
+
require "protobuf"
|
4
|
+
# We don't need this, but the CLI attempts to terminate.
|
5
|
+
require "protobuf/rpc/service_directory"
|
6
|
+
|
7
|
+
require "connection_pool"
|
8
|
+
|
9
|
+
require "nats/io/client"
|
10
|
+
|
11
|
+
require "protobuf/nats/client"
|
12
|
+
require "protobuf/nats/server"
|
13
|
+
require "protobuf/nats/runner"
|
14
|
+
require "protobuf/nats/config"
|
15
|
+
|
16
|
+
module Protobuf
|
17
|
+
module Nats
|
18
|
+
class << self
|
19
|
+
attr_accessor :client_nats_connection
|
20
|
+
end
|
21
|
+
|
22
|
+
module Messages
|
23
|
+
ACK = "\1".freeze
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.config
|
27
|
+
@config ||= begin
|
28
|
+
config = ::Protobuf::Nats::Config.new
|
29
|
+
config.load_from_yml
|
30
|
+
config
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Eagerly load the yml config.
|
35
|
+
config
|
36
|
+
|
37
|
+
def self.subscription_key(service_klass, service_method)
|
38
|
+
service_class_name = service_klass.name.underscore.gsub("/", ".")
|
39
|
+
service_method_name = service_method.to_s.underscore
|
40
|
+
"rpc.#{service_class_name}.#{service_method_name}"
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.start_client_nats_connection
|
44
|
+
@client_nats_connection = ::NATS::IO::Client.new
|
45
|
+
@client_nats_connection.connect(config.connection_options)
|
46
|
+
|
47
|
+
# Ensure we have a valid connection to the NATS server.
|
48
|
+
@client_nats_connection.flush(5)
|
49
|
+
|
50
|
+
true
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.ensure_client_nats_connection_was_started
|
54
|
+
@ensure_client_nats_connection_was_started ||= start_client_nats_connection
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'protobuf/nats/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "protobuf-nats"
|
8
|
+
spec.version = Protobuf::Nats::VERSION
|
9
|
+
spec.authors = ["Brandon Dewitt"]
|
10
|
+
spec.email = ["brandonsdewitt@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{ ruby-protobuf client/server for nats }
|
13
|
+
spec.description = %q{ ruby-protobuf client/server for nats }
|
14
|
+
#spec.homepage = "TODO: Put your gem's website or public repo URL here."
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
if spec.respond_to?(:metadata)
|
20
|
+
spec.metadata['allowed_push_host'] = "https://rubygems.org"
|
21
|
+
else
|
22
|
+
raise "RubyGems 2.0 or newer is required to protect against " \
|
23
|
+
"public gem pushes."
|
24
|
+
end
|
25
|
+
|
26
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
27
|
+
f.match(%r{^(test|spec|features)/})
|
28
|
+
end
|
29
|
+
spec.bindir = "exe"
|
30
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
|
+
spec.require_paths = ["lib"]
|
32
|
+
|
33
|
+
spec.add_runtime_dependency "protobuf", "3.7.0.pre3"
|
34
|
+
spec.add_runtime_dependency "connection_pool"
|
35
|
+
spec.add_runtime_dependency "concurrent-ruby"
|
36
|
+
spec.add_runtime_dependency "nats-pure"
|
37
|
+
|
38
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
39
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
40
|
+
spec.add_development_dependency "rspec"
|
41
|
+
spec.add_development_dependency "benchmark-ips"
|
42
|
+
spec.add_development_dependency "pry"
|
43
|
+
end
|
metadata
ADDED
@@ -0,0 +1,194 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: protobuf-nats
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Brandon Dewitt
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-03-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: protobuf
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 3.7.0.pre3
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 3.7.0.pre3
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: connection_pool
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: concurrent-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: nats-pure
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: bundler
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '1.14'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '1.14'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '10.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '10.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: benchmark-ips
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pry
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: " ruby-protobuf client/server for nats "
|
140
|
+
email:
|
141
|
+
- brandonsdewitt@gmail.com
|
142
|
+
executables: []
|
143
|
+
extensions: []
|
144
|
+
extra_rdoc_files: []
|
145
|
+
files:
|
146
|
+
- ".gitignore"
|
147
|
+
- ".travis.yml"
|
148
|
+
- Gemfile
|
149
|
+
- LICENSE.txt
|
150
|
+
- README.md
|
151
|
+
- Rakefile
|
152
|
+
- bench/client.rb
|
153
|
+
- bench/real_client.rb
|
154
|
+
- bench/real_server.sh
|
155
|
+
- bench/server.rb
|
156
|
+
- bin/console
|
157
|
+
- bin/setup
|
158
|
+
- examples/warehouse/app.rb
|
159
|
+
- examples/warehouse/start_client.sh
|
160
|
+
- examples/warehouse/start_server.sh
|
161
|
+
- lib/protobuf/nats.rb
|
162
|
+
- lib/protobuf/nats/client.rb
|
163
|
+
- lib/protobuf/nats/config.rb
|
164
|
+
- lib/protobuf/nats/fixed_thread_pool_with_no_queue.rb
|
165
|
+
- lib/protobuf/nats/runner.rb
|
166
|
+
- lib/protobuf/nats/server.rb
|
167
|
+
- lib/protobuf/nats/version.rb
|
168
|
+
- protobuf-nats.gemspec
|
169
|
+
homepage:
|
170
|
+
licenses:
|
171
|
+
- MIT
|
172
|
+
metadata:
|
173
|
+
allowed_push_host: https://rubygems.org
|
174
|
+
post_install_message:
|
175
|
+
rdoc_options: []
|
176
|
+
require_paths:
|
177
|
+
- lib
|
178
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
179
|
+
requirements:
|
180
|
+
- - ">="
|
181
|
+
- !ruby/object:Gem::Version
|
182
|
+
version: '0'
|
183
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
requirements: []
|
189
|
+
rubyforge_project:
|
190
|
+
rubygems_version: 2.5.2
|
191
|
+
signing_key:
|
192
|
+
specification_version: 4
|
193
|
+
summary: ruby-protobuf client/server for nats
|
194
|
+
test_files: []
|