sanger_warren 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.rubocop.yml +21 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +10 -0
- data/Gemfile +13 -0
- data/Gemfile.lock +77 -0
- data/LICENSE.txt +27 -0
- data/README.md +81 -0
- data/Rakefile +8 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/checksums +0 -0
- data/complete +0 -0
- data/lib/warren.rb +29 -0
- data/lib/warren/callback.rb +64 -0
- data/lib/warren/callback/broadcast_associated_with_warren.rb +39 -0
- data/lib/warren/callback/broadcast_with_warren.rb +33 -0
- data/lib/warren/handler.rb +14 -0
- data/lib/warren/handler/broadcast.rb +132 -0
- data/lib/warren/handler/log.rb +69 -0
- data/lib/warren/handler/test.rb +111 -0
- data/lib/warren/message.rb +15 -0
- data/lib/warren/message/full.rb +26 -0
- data/lib/warren/message/short.rb +22 -0
- data/lib/warren/version.rb +5 -0
- data/object_types +0 -0
- data/objects/root.dat +0 -0
- data/proxy_types +0 -0
- data/sanger-warren.gemspec +38 -0
- metadata +119 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: '08237db3ed2d846148e3e1b127b737d660cb48f755489ced7fbda20d582fb3c2'
|
4
|
+
data.tar.gz: 41dc24f4262cc1b641e985058d7ef6b1cb294ef9e26c42775ac68bd21182c8e3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: e60a34f8902db181b06978f0b554485c8f69148e9111b2a325ad6b37b13dc4abef93f5e8d010ad4e2d2d7dab4dcdfddfa51f6b5fe5866e037d9d437e85095af2
|
7
|
+
data.tar.gz: 98a52f7cdfaf2815f6b83a5166791bb83ab84aaa37e87ed9b4d68534fccb7a9efe14ec0c618467bd24f8c67deecd2124833245b0e3bc7b156ca48ff2a4762bb6
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# The behavior of RuboCop can be controlled via the .rubocop.yml
|
2
|
+
# configuration file. It makes it possible to enable/disable
|
3
|
+
# certain cops (checks) and to alter their behavior if they accept
|
4
|
+
# any parameters. The file can be placed either in your home
|
5
|
+
# directory or in some project directory.
|
6
|
+
#
|
7
|
+
# RuboCop will start looking for the configuration file in the directory
|
8
|
+
# where the inspected file is and continue its way up to the root directory.
|
9
|
+
#
|
10
|
+
# See https://docs.rubocop.org/rubocop/configuration
|
11
|
+
AllCops:
|
12
|
+
TargetRubyVersion: 2.6
|
13
|
+
# Blocks are used to structure tests and are part of the shoulda dsl.
|
14
|
+
# The standard BlockLength limits are too stringent for this purpose.
|
15
|
+
# Block DSL is also used in routes
|
16
|
+
# We should be careful keeping rake tasks here, as large tasks are still an issue.
|
17
|
+
# Not sure if we need to split namespaces across several files though.
|
18
|
+
# We exclude builders, as they are concerned with rendering xml
|
19
|
+
Metrics/BlockLength:
|
20
|
+
ExcludedMethods:
|
21
|
+
- describe
|
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
sanger_warren (0.1.0)
|
5
|
+
bunny (~> 2.17.0)
|
6
|
+
connection_pool (~> 2.2.0)
|
7
|
+
multi_json (~> 1.0)
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
amq-protocol (2.3.2)
|
13
|
+
ast (2.4.1)
|
14
|
+
bunny (2.17.0)
|
15
|
+
amq-protocol (~> 2.3, >= 2.3.1)
|
16
|
+
coderay (1.1.2)
|
17
|
+
connection_pool (2.2.3)
|
18
|
+
diff-lcs (1.4.4)
|
19
|
+
docile (1.3.2)
|
20
|
+
method_source (0.9.2)
|
21
|
+
multi_json (1.13.1)
|
22
|
+
parallel (1.19.2)
|
23
|
+
parser (2.7.2.0)
|
24
|
+
ast (~> 2.4.1)
|
25
|
+
pry (0.12.2)
|
26
|
+
coderay (~> 1.1.0)
|
27
|
+
method_source (~> 0.9.0)
|
28
|
+
rainbow (3.0.0)
|
29
|
+
rake (12.3.3)
|
30
|
+
regexp_parser (1.8.2)
|
31
|
+
rexml (3.2.4)
|
32
|
+
rspec (3.9.0)
|
33
|
+
rspec-core (~> 3.9.0)
|
34
|
+
rspec-expectations (~> 3.9.0)
|
35
|
+
rspec-mocks (~> 3.9.0)
|
36
|
+
rspec-core (3.9.3)
|
37
|
+
rspec-support (~> 3.9.3)
|
38
|
+
rspec-expectations (3.9.2)
|
39
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
40
|
+
rspec-support (~> 3.9.0)
|
41
|
+
rspec-mocks (3.9.1)
|
42
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
43
|
+
rspec-support (~> 3.9.0)
|
44
|
+
rspec-support (3.9.3)
|
45
|
+
rubocop (1.0.0)
|
46
|
+
parallel (~> 1.10)
|
47
|
+
parser (>= 2.7.1.5)
|
48
|
+
rainbow (>= 2.2.2, < 4.0)
|
49
|
+
regexp_parser (>= 1.8)
|
50
|
+
rexml
|
51
|
+
rubocop-ast (>= 0.6.0)
|
52
|
+
ruby-progressbar (~> 1.7)
|
53
|
+
unicode-display_width (>= 1.4.0, < 2.0)
|
54
|
+
rubocop-ast (1.0.0)
|
55
|
+
parser (>= 2.7.1.5)
|
56
|
+
ruby-progressbar (1.10.1)
|
57
|
+
simplecov (0.19.0)
|
58
|
+
docile (~> 1.1)
|
59
|
+
simplecov-html (~> 0.11)
|
60
|
+
simplecov-html (0.12.3)
|
61
|
+
unicode-display_width (1.7.0)
|
62
|
+
yard (0.9.25)
|
63
|
+
|
64
|
+
PLATFORMS
|
65
|
+
ruby
|
66
|
+
|
67
|
+
DEPENDENCIES
|
68
|
+
pry
|
69
|
+
rake (~> 12.0)
|
70
|
+
rspec (~> 3.0)
|
71
|
+
rubocop
|
72
|
+
sanger_warren!
|
73
|
+
simplecov
|
74
|
+
yard
|
75
|
+
|
76
|
+
BUNDLED WITH
|
77
|
+
2.1.4
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
|
2
|
+
Authors:
|
3
|
+
James Glover <james.glover@sanger.ac.uk>
|
4
|
+
|
5
|
+
This program is free software; you can redistribute it and/or modify
|
6
|
+
it under the terms of the GNU General Public License as published by
|
7
|
+
the Free Software Foundation; either version 1, or (at your option)
|
8
|
+
any later version.
|
9
|
+
|
10
|
+
This program is distributed in the hope that it will be useful,
|
11
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
13
|
+
GNU General Public License for more details.
|
14
|
+
|
15
|
+
You should have received a copy of the GNU General Public License
|
16
|
+
along with this program; if not, see <http://www.gnu.org/licenses/>.
|
17
|
+
|
18
|
+
|
19
|
+
The usage of a range of years within a copyright statement contained within
|
20
|
+
this distribution should be interpreted as being equivalent to a list of years
|
21
|
+
including the first and last year specified and all consecutive years between
|
22
|
+
them. For example, a copyright statement that reads "Copyright (c) 2005, 2007-
|
23
|
+
2009, 2011-2012" should be interpreted as being identical to a statement that
|
24
|
+
reads "Copyright (c) 2005, 2007, 2008, 2009, 2011, 2012" and a copyright
|
25
|
+
statement that reads "Copyright (c) 2005-2012" should be interpreted as being
|
26
|
+
identical to a statement that reads "Copyright (c) 2005, 2006, 2007, 2008, 2009
|
27
|
+
, 2010 , 2011 , 2012".
|
data/README.md
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
# Warren
|
2
|
+
|
3
|
+
Warren extracts the connection pooling behaviour originally in Sequencescape
|
4
|
+
in order to provide a common interface for our Rails app interaction with
|
5
|
+
RabbitMQ via the bunny gem.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'sanger-warren'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle install
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install sanger-warren
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
If using with a Rails app, here's an example to get you started:
|
26
|
+
|
27
|
+
```ruby
|
28
|
+
# config/initializers/warren.rb
|
29
|
+
require 'warren'
|
30
|
+
|
31
|
+
Warren.setup(Rails.application.config.warren.deep_symbolize_keys.slice(:type, :config))
|
32
|
+
```
|
33
|
+
|
34
|
+
```ruby
|
35
|
+
# config/application.rb
|
36
|
+
# ...
|
37
|
+
config.warren = config_for(:warren)
|
38
|
+
# ...
|
39
|
+
```
|
40
|
+
|
41
|
+
```yaml
|
42
|
+
# config/warren.yml
|
43
|
+
development:
|
44
|
+
type: log
|
45
|
+
config: # Useful to allow easy switching to broadcast in development
|
46
|
+
routing_key_prefix: 'dev'
|
47
|
+
server:
|
48
|
+
host: localhost
|
49
|
+
port: 5672
|
50
|
+
username: guest
|
51
|
+
password: guest
|
52
|
+
vhost: /
|
53
|
+
frame_max: 0
|
54
|
+
heartbeat: 30
|
55
|
+
exchange: exchange_to_use
|
56
|
+
test:
|
57
|
+
type: test
|
58
|
+
production: # In practice keep this out of your source control
|
59
|
+
type: broadcast
|
60
|
+
config: # Useful to allow easy switching to broadcast
|
61
|
+
routing_key_prefix: 'production'
|
62
|
+
server:
|
63
|
+
host: localhost # Or a remote host
|
64
|
+
port: 5672
|
65
|
+
username: ...
|
66
|
+
password: ...
|
67
|
+
vhost: /
|
68
|
+
frame_max: 0
|
69
|
+
heartbeat: 30
|
70
|
+
exchange: exchange_to_use
|
71
|
+
```
|
72
|
+
|
73
|
+
## Development
|
74
|
+
|
75
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
76
|
+
|
77
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
78
|
+
|
79
|
+
## Contributing
|
80
|
+
|
81
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/Warren.
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'bundler/setup'
|
5
|
+
require 'Warren'
|
6
|
+
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
9
|
+
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
11
|
+
# require "pry"
|
12
|
+
# Pry.start
|
13
|
+
|
14
|
+
require 'irb'
|
15
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/checksums
ADDED
File without changes
|
data/complete
ADDED
File without changes
|
data/lib/warren.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'warren/version'
|
4
|
+
require 'warren/callback'
|
5
|
+
require 'warren/handler'
|
6
|
+
require 'warren/message'
|
7
|
+
|
8
|
+
#
|
9
|
+
# Module Warren provides connection pooling for RabbitMQ Connections
|
10
|
+
#
|
11
|
+
module Warren
|
12
|
+
def self.construct(type:, config: {})
|
13
|
+
case type
|
14
|
+
when 'test' then Warren::Handler::Test.new
|
15
|
+
when 'log' then Warren::Handler::Log.new(logger: config.fetch(:logger) { Rails.logger })
|
16
|
+
when 'broadcast' then Warren::Handler::Broadcast.new(config)
|
17
|
+
else raise StandardError, "Unknown type warren: #{type}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.setup(opts, logger: Rails.logger)
|
22
|
+
logger.warn 'Recreating Warren handler when one already exists' if handler.present?
|
23
|
+
@handler = construct(opts.symbolize_keys)
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.handler
|
27
|
+
@handler
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'message/full'
|
4
|
+
require 'connection_pool'
|
5
|
+
|
6
|
+
require_relative 'callback/broadcast_with_warren'
|
7
|
+
require_relative 'callback/broadcast_associated_with_warren'
|
8
|
+
#
|
9
|
+
# Module Warren::Callback provides methods to assist with
|
10
|
+
# setting up message broadcast
|
11
|
+
#
|
12
|
+
module Warren
|
13
|
+
#
|
14
|
+
# Module Warren::BroadcastMessages provides methods to assist with
|
15
|
+
# setting up message broadcast
|
16
|
+
#
|
17
|
+
module Callback
|
18
|
+
# Provides the broadcast_with_warren and broadcasts_associated_with_warren to
|
19
|
+
# ActiveRecord::Base classes to configure broadcast
|
20
|
+
module ClassMethods
|
21
|
+
attr_reader :associated_to_broadcast, :warren
|
22
|
+
|
23
|
+
#
|
24
|
+
# Records of this type are broadcast via RabbitMQ when a transaction is closed.
|
25
|
+
#
|
26
|
+
# @return [void]
|
27
|
+
#
|
28
|
+
def broadcast_with_warren(handler: Warren.handler)
|
29
|
+
after_commit BroadcastWithWarren.new(handler: handler)
|
30
|
+
end
|
31
|
+
|
32
|
+
#
|
33
|
+
# When records of this type are saved, broadcast the associated records once
|
34
|
+
# the transaction is closed. (Requires that associated record is broadcast_with_warren)
|
35
|
+
#
|
36
|
+
# @param [Symbol,Array<Symbol>] associated One or more symbols indicating the associations to broadcast.
|
37
|
+
#
|
38
|
+
# @return [void]
|
39
|
+
#
|
40
|
+
def broadcasts_associated_with_warren(*associated, handler: Warren.handler)
|
41
|
+
after_save BroadcastAssociatedWithWarren.new(associated, handler: handler)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.included(base)
|
46
|
+
base.class_eval do
|
47
|
+
extend ClassMethods
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def broadcast
|
52
|
+
# This results in borrowing a connection from the pool
|
53
|
+
# per-message. Which isn't ideal. Ideally we'd either
|
54
|
+
# check out a connection per thread or per transaction.
|
55
|
+
# Any checked out connections will need to be checked back
|
56
|
+
# in once the thread/transaction ends with high reliability.
|
57
|
+
# So we're doing things the easy way until:
|
58
|
+
# 1) Performance requires something more complicated
|
59
|
+
# 2) We find a way to achieve the above without monkey-patching
|
60
|
+
# or other complexity (Its probably possible)
|
61
|
+
Warren.handler << Warren::Message::Full.new(self)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Warren
|
4
|
+
module Callback
|
5
|
+
# Warren::Callback::BroadcastWithWarren is a Callback class
|
6
|
+
# which is used to handle message broadcast of records associated with
|
7
|
+
# ActiveRecord::Base objects on save. Associated records will be queued for
|
8
|
+
# broadcast when the transaction is closed.
|
9
|
+
# @see https://guides.rubyonrails.org/active_record_callbacks.html#callback-classes
|
10
|
+
class BroadcastAssociatedWithWarren
|
11
|
+
attr_reader :handler, :associations
|
12
|
+
|
13
|
+
#
|
14
|
+
# Creates the callback object
|
15
|
+
#
|
16
|
+
# @param handler [Warren::Handler] The handler to take the messaged (unused)
|
17
|
+
# @param associations [Array<Symbol>] An array of symbols reflecting the
|
18
|
+
# names of associations
|
19
|
+
#
|
20
|
+
def initialize(associations, handler:)
|
21
|
+
@associations = associations
|
22
|
+
@handler = handler
|
23
|
+
end
|
24
|
+
|
25
|
+
#
|
26
|
+
# After save callback: configured via broadcasts_associated_with_warren
|
27
|
+
# Adds any associated records to the transaction, ensuring their after commit
|
28
|
+
# methods will fire.
|
29
|
+
#
|
30
|
+
# @return [void]
|
31
|
+
#
|
32
|
+
def after_save(record)
|
33
|
+
associations.each do |association|
|
34
|
+
record.send(association).try(:add_to_transaction)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'warren/message/short'
|
4
|
+
|
5
|
+
module Warren
|
6
|
+
module Callback
|
7
|
+
# Warren::Callback::BroadcastWithWarren is a Callback class
|
8
|
+
# which is used to handle message broadcast of ActiveRecord::Base objects
|
9
|
+
# on commit
|
10
|
+
# @see https://guides.rubyonrails.org/active_record_callbacks.html#callback-classes
|
11
|
+
class BroadcastWithWarren
|
12
|
+
attr_reader :handler, :message_class
|
13
|
+
|
14
|
+
#
|
15
|
+
# Creates the callback object
|
16
|
+
#
|
17
|
+
# @param handler [Warren::Handler] The handler to take the messaged
|
18
|
+
# @param message_class [Warren::Message] The adpater to render the messages
|
19
|
+
#
|
20
|
+
def initialize(handler:, message_class: Warren::Message::Short)
|
21
|
+
@handler = handler
|
22
|
+
@message_class = message_class
|
23
|
+
end
|
24
|
+
|
25
|
+
def after_commit(record)
|
26
|
+
# Message rendering is slow in some cases. This broadcasts an initial
|
27
|
+
# lightweight message which can be picked up and rendered asynchronously
|
28
|
+
# Borrows connection as per #broadcast
|
29
|
+
handler << message_class.new(record)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'handler/broadcast'
|
4
|
+
require_relative 'handler/log'
|
5
|
+
require_relative 'handler/test'
|
6
|
+
module Warren
|
7
|
+
# A {Warren::Handler} provides an interface for sending messages to either
|
8
|
+
# a message queue, a log, or an internal store for testing purposes.
|
9
|
+
module Handler
|
10
|
+
def self.routing_key_template(prefix)
|
11
|
+
prefix ? "#{prefix}.%s" : '%s'
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,132 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'bunny'
|
4
|
+
|
5
|
+
module Warren
|
6
|
+
module Handler
|
7
|
+
#
|
8
|
+
# Class Warren::Broadcast provides a connection pool of
|
9
|
+
# threadsafe RabbitMQ channels for broadcasting messages
|
10
|
+
#
|
11
|
+
class Broadcast
|
12
|
+
# Wraps a {Bunny::Channel}
|
13
|
+
class Channel
|
14
|
+
def initialize(bun_channel, routing_key_template:, exchange: nil)
|
15
|
+
@bun_channel = bun_channel
|
16
|
+
@exchange_name = exchange
|
17
|
+
@routing_key_template = routing_key_template
|
18
|
+
end
|
19
|
+
|
20
|
+
def <<(message)
|
21
|
+
exchange.publish(message.payload, routing_key: key_for(message))
|
22
|
+
self
|
23
|
+
end
|
24
|
+
|
25
|
+
def close
|
26
|
+
@bun_channel.close
|
27
|
+
end
|
28
|
+
|
29
|
+
private
|
30
|
+
|
31
|
+
def exchange
|
32
|
+
raise StandardError, 'No exchange configured' if @exchange_name.nil?
|
33
|
+
|
34
|
+
@exchange ||= @bun_channel.topic(@exchange_name, auto_delete: false, durable: true)
|
35
|
+
end
|
36
|
+
|
37
|
+
def key_for(message)
|
38
|
+
@routing_key_template % message.routing_key
|
39
|
+
end
|
40
|
+
end
|
41
|
+
#
|
42
|
+
# Creates a warren but does not connect.
|
43
|
+
#
|
44
|
+
# @param [Hash] server Server config options passes straight to Bunny
|
45
|
+
# @param [String] exchange The name of the exchange to connect to
|
46
|
+
# @param [Integer] pool_size The connection pool size
|
47
|
+
# @param [String,nil] routing_key_prefix The prefix to pass before the routing key.
|
48
|
+
# Can be used to ensure environments remain distinct.
|
49
|
+
def initialize(exchange:, routing_key_prefix:, server: {}, pool_size: 14)
|
50
|
+
@server = server
|
51
|
+
@exchange_name = exchange
|
52
|
+
@pool_size = pool_size
|
53
|
+
@routing_key_template = Handler.routing_key_template(routing_key_prefix)
|
54
|
+
end
|
55
|
+
|
56
|
+
#
|
57
|
+
# Opens a connection to the RabbitMQ server. Will need to be re-initialized after forking.
|
58
|
+
#
|
59
|
+
# @return [true] We've connected!
|
60
|
+
#
|
61
|
+
def connect
|
62
|
+
reset_pool
|
63
|
+
start_session
|
64
|
+
end
|
65
|
+
|
66
|
+
#
|
67
|
+
# Closes the connection. Call before forking to avoid leaking connections
|
68
|
+
#
|
69
|
+
#
|
70
|
+
# @return [true] We've disconnected
|
71
|
+
#
|
72
|
+
def disconnect
|
73
|
+
close_session
|
74
|
+
end
|
75
|
+
|
76
|
+
#
|
77
|
+
# Yields an exchange which gets returned to the pool on block closure
|
78
|
+
#
|
79
|
+
#
|
80
|
+
# @return [void]
|
81
|
+
#
|
82
|
+
# @yieldreturn [Warren::Broadcast::Channel] A rabbitMQ channel that sends messages to the configured exchange
|
83
|
+
def with_channel(&block)
|
84
|
+
connection_pool.with(&block)
|
85
|
+
end
|
86
|
+
|
87
|
+
#
|
88
|
+
# Borrows a RabbitMQ channel, sends a message, and immediately returns it again.
|
89
|
+
# Useful if you only need to send one message.
|
90
|
+
#
|
91
|
+
# @param [Warren::Message] message The message to broadcast. Must respond to #routing_key and #payload
|
92
|
+
#
|
93
|
+
# @return [Warren::Broadcast] Returns itself to allow chaining. But you're
|
94
|
+
# probably better off using #with_channel
|
95
|
+
# in that case
|
96
|
+
#
|
97
|
+
def <<(message)
|
98
|
+
with_channel { |channel| channel << message }
|
99
|
+
self
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def session
|
105
|
+
@session ||= Bunny.new(@server)
|
106
|
+
end
|
107
|
+
|
108
|
+
def connection_pool
|
109
|
+
@connection_pool ||= start_session && ConnectionPool.new(size: @pool_size, timeout: 5) do
|
110
|
+
Channel.new(session.create_channel, exchange: @exchange_name,
|
111
|
+
routing_key_template: @routing_key_template)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def start_session
|
116
|
+
session.start
|
117
|
+
true
|
118
|
+
end
|
119
|
+
|
120
|
+
def close_session
|
121
|
+
reset_pool
|
122
|
+
@session&.close
|
123
|
+
@session = nil
|
124
|
+
end
|
125
|
+
|
126
|
+
def reset_pool
|
127
|
+
@connection_pool&.shutdown { |ch| ch.close }
|
128
|
+
@connection_pool = nil
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Warren
|
4
|
+
module Handler
|
5
|
+
# Class Warren::Log provides a dummy RabbitMQ
|
6
|
+
# connection pool for use during development
|
7
|
+
class Log
|
8
|
+
# Mimics a {Bunny::Channel} but instead passes out to a logger
|
9
|
+
class Channel
|
10
|
+
def initialize(logger, routing_key_template: '%s')
|
11
|
+
@logger = logger
|
12
|
+
@routing_key_template = routing_key_template
|
13
|
+
end
|
14
|
+
|
15
|
+
def <<(message)
|
16
|
+
@logger.info "Published: #{key_for(message)}"
|
17
|
+
@logger.debug "Payload: #{message.payload}"
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def key_for(message)
|
24
|
+
@routing_key_template % message.routing_key
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
attr_reader :logger
|
29
|
+
|
30
|
+
def initialize(logger:, routing_key_prefix: nil)
|
31
|
+
@logger = logger
|
32
|
+
@routing_key_template = Handler.routing_key_template(routing_key_prefix)
|
33
|
+
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Provide API compatibility with the RabbitMQ versions
|
37
|
+
# Do nothing in this case
|
38
|
+
#
|
39
|
+
def connect; end
|
40
|
+
|
41
|
+
def disconnect; end
|
42
|
+
|
43
|
+
#
|
44
|
+
# Yields a Warren::Log::Channel
|
45
|
+
#
|
46
|
+
#
|
47
|
+
# @return [void]
|
48
|
+
#
|
49
|
+
# @yieldreturn [Warren::Log::Channel] A rabbitMQ channel that logs messaged to the test warren
|
50
|
+
def with_channel
|
51
|
+
yield Channel.new(@logger, routing_key_template: @routing_key_template)
|
52
|
+
end
|
53
|
+
|
54
|
+
#
|
55
|
+
# Sends a message to the log channel.
|
56
|
+
# Useful if you only need to send one message.
|
57
|
+
#
|
58
|
+
# @param [Warren::Message] message The message to broadcast. Must respond to #routing_key and #payload
|
59
|
+
#
|
60
|
+
# @return [Warren::Log] Returns itself to allow chaining. But you're
|
61
|
+
# probably better off using #with_channel
|
62
|
+
# in that case
|
63
|
+
#
|
64
|
+
def <<(message)
|
65
|
+
with_channel { |c| c << message }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Warren
|
4
|
+
module Handler
|
5
|
+
# Class Warren::Test provides provides a dummy RabbitMQ
|
6
|
+
# connection pool for use during testing
|
7
|
+
class Test
|
8
|
+
DISABLED_WARNING = <<~DISABLED_WARREN
|
9
|
+
Test made against a disabled warren.
|
10
|
+
Warren::Handler::Test must be explicitly enabled to track messages,
|
11
|
+
it is a good idea to disable it again after testing the relevant
|
12
|
+
behaviour. This ensures we track messages on a per-test basis, and
|
13
|
+
avoids unnecessary message storage.
|
14
|
+
|
15
|
+
If using rspec it is suggested that you add the following to your
|
16
|
+
spec_helper.rb
|
17
|
+
|
18
|
+
config.around(:each, warren: true) do |ex|
|
19
|
+
Warren.handler.enable!
|
20
|
+
ex.run
|
21
|
+
Warren.handler.disable!
|
22
|
+
end
|
23
|
+
|
24
|
+
You can then tag tests with warren: true to enable warren testing.
|
25
|
+
DISABLED_WARREN
|
26
|
+
# Stand in for {Bunny::Channel}, provides a store of messages to use
|
27
|
+
# in test assertions
|
28
|
+
class Channel
|
29
|
+
def initialize(warren)
|
30
|
+
@warren = warren
|
31
|
+
end
|
32
|
+
|
33
|
+
def <<(message)
|
34
|
+
@warren << message
|
35
|
+
end
|
36
|
+
end
|
37
|
+
#
|
38
|
+
# Creates a test warren with no messages.
|
39
|
+
# Test warrens are shared across all threads.
|
40
|
+
#
|
41
|
+
# @param [_] _args Configuration arguments are ignored.
|
42
|
+
#
|
43
|
+
def initialize(*_args)
|
44
|
+
@messages = []
|
45
|
+
@enabled = false
|
46
|
+
end
|
47
|
+
|
48
|
+
#
|
49
|
+
# Provide API compatibility with the RabbitMQ versions
|
50
|
+
# Do nothing in this case
|
51
|
+
#
|
52
|
+
def connect; end
|
53
|
+
|
54
|
+
def disconnect; end
|
55
|
+
|
56
|
+
#
|
57
|
+
# Yields an exchange which gets returned to the pool on block closure
|
58
|
+
#
|
59
|
+
#
|
60
|
+
# @return [void]
|
61
|
+
#
|
62
|
+
# @yieldreturn [Warren::Test::Channel] A rabbitMQ channel that logs messaged to the test warren
|
63
|
+
def with_channel
|
64
|
+
yield Channel.new(self)
|
65
|
+
end
|
66
|
+
|
67
|
+
def clear_messages
|
68
|
+
@messages = []
|
69
|
+
end
|
70
|
+
|
71
|
+
def last_message
|
72
|
+
messages.last
|
73
|
+
end
|
74
|
+
|
75
|
+
def message_count
|
76
|
+
messages.length
|
77
|
+
end
|
78
|
+
|
79
|
+
def messages_matching(routing_key)
|
80
|
+
messages.count { |message| message.routing_key == routing_key }
|
81
|
+
end
|
82
|
+
|
83
|
+
def enable!
|
84
|
+
@enabled = true
|
85
|
+
clear_messages
|
86
|
+
end
|
87
|
+
|
88
|
+
# Clean up after ourselves to avoid memory leaks
|
89
|
+
def disable!
|
90
|
+
@enabled = false
|
91
|
+
clear_messages
|
92
|
+
end
|
93
|
+
|
94
|
+
def messages
|
95
|
+
raise_if_not_tracking
|
96
|
+
@messages
|
97
|
+
end
|
98
|
+
|
99
|
+
# Disable message logging if not required
|
100
|
+
def <<(message)
|
101
|
+
@messages << message if @enabled
|
102
|
+
end
|
103
|
+
|
104
|
+
private
|
105
|
+
|
106
|
+
def raise_if_not_tracking
|
107
|
+
raise StandardError, DISABLED_WARNING unless @enabled
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# Namespace to collect message formats
|
4
|
+
# A Warren compatible message must implement:
|
5
|
+
# routing_key: returns the routing_key for the message
|
6
|
+
# payloadL returns the message payload
|
7
|
+
|
8
|
+
require_relative 'message/short'
|
9
|
+
require_relative 'message/full'
|
10
|
+
|
11
|
+
# Additionally, if you wish to use the Message with the ActiveRecord
|
12
|
+
# helpers, then the initialize should take the ActiveRecord::Base object
|
13
|
+
# as a single argument
|
14
|
+
module Message
|
15
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Warren
|
4
|
+
module Message
|
5
|
+
# Generates a payload of an active_record object
|
6
|
+
class Full
|
7
|
+
attr_reader :record
|
8
|
+
|
9
|
+
def initialize(record)
|
10
|
+
@record = record
|
11
|
+
end
|
12
|
+
|
13
|
+
def routing_key
|
14
|
+
if record.respond_to?(:routing_key)
|
15
|
+
record.routing_key
|
16
|
+
else
|
17
|
+
"saved.#{record.class.name.underscore}.#{record.id}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def payload
|
22
|
+
MultiJson.dump(record)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Warren
|
4
|
+
module Message
|
5
|
+
# Light-weight interim message which can be expanded to a full payload later.
|
6
|
+
class Short
|
7
|
+
attr_reader :record
|
8
|
+
|
9
|
+
def initialize(record)
|
10
|
+
@record = record
|
11
|
+
end
|
12
|
+
|
13
|
+
def routing_key
|
14
|
+
"queue_broadcast.#{record.class.name.underscore}.#{record.id}"
|
15
|
+
end
|
16
|
+
|
17
|
+
def payload
|
18
|
+
[record.class.name, record.id].to_json
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/object_types
ADDED
Binary file
|
data/objects/root.dat
ADDED
Binary file
|
data/proxy_types
ADDED
Binary file
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'lib/warren/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = 'sanger_warren'
|
7
|
+
spec.version = Warren::VERSION
|
8
|
+
spec.authors = ['James Glover']
|
9
|
+
spec.email = ['james.glover@sanger.ac.uk']
|
10
|
+
|
11
|
+
spec.summary = 'Configuring and managing bunny RabbitMQ connections'
|
12
|
+
spec.description = <<~DESCRIPTION
|
13
|
+
Warren provides connection pooling for RabbitMQ connections. It also adds
|
14
|
+
the ability to switch in different adapters during testing and development.
|
15
|
+
DESCRIPTION
|
16
|
+
spec.homepage = 'https://github.com/sanger/warren'
|
17
|
+
spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
|
18
|
+
|
19
|
+
# spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
|
20
|
+
|
21
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
22
|
+
spec.metadata['source_code_uri'] = 'https://github.com/sanger/warren'
|
23
|
+
spec.metadata['changelog_uri'] = 'https://github.com/sanger/warren/blob/master/CHANGELOG.md'
|
24
|
+
|
25
|
+
# Specify which files should be added to the gem when it is released.
|
26
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
27
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
28
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
29
|
+
end
|
30
|
+
spec.bindir = 'exe'
|
31
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
32
|
+
spec.require_paths = ['lib']
|
33
|
+
|
34
|
+
# Dependencies
|
35
|
+
spec.add_runtime_dependency 'bunny', '~> 2.17.0'
|
36
|
+
spec.add_runtime_dependency 'connection_pool', '~> 2.2.0'
|
37
|
+
spec.add_runtime_dependency 'multi_json', '~> 1.0'
|
38
|
+
end
|
metadata
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sanger_warren
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- James Glover
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-10-30 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bunny
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 2.17.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 2.17.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: connection_pool
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 2.2.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 2.2.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: multi_json
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.0'
|
55
|
+
description: |
|
56
|
+
Warren provides connection pooling for RabbitMQ connections. It also adds
|
57
|
+
the ability to switch in different adapters during testing and development.
|
58
|
+
email:
|
59
|
+
- james.glover@sanger.ac.uk
|
60
|
+
executables: []
|
61
|
+
extensions: []
|
62
|
+
extra_rdoc_files: []
|
63
|
+
files:
|
64
|
+
- ".gitignore"
|
65
|
+
- ".rspec"
|
66
|
+
- ".rubocop.yml"
|
67
|
+
- ".travis.yml"
|
68
|
+
- CHANGELOG.md
|
69
|
+
- Gemfile
|
70
|
+
- Gemfile.lock
|
71
|
+
- LICENSE.txt
|
72
|
+
- README.md
|
73
|
+
- Rakefile
|
74
|
+
- bin/console
|
75
|
+
- bin/setup
|
76
|
+
- checksums
|
77
|
+
- complete
|
78
|
+
- lib/warren.rb
|
79
|
+
- lib/warren/callback.rb
|
80
|
+
- lib/warren/callback/broadcast_associated_with_warren.rb
|
81
|
+
- lib/warren/callback/broadcast_with_warren.rb
|
82
|
+
- lib/warren/handler.rb
|
83
|
+
- lib/warren/handler/broadcast.rb
|
84
|
+
- lib/warren/handler/log.rb
|
85
|
+
- lib/warren/handler/test.rb
|
86
|
+
- lib/warren/message.rb
|
87
|
+
- lib/warren/message/full.rb
|
88
|
+
- lib/warren/message/short.rb
|
89
|
+
- lib/warren/version.rb
|
90
|
+
- object_types
|
91
|
+
- objects/root.dat
|
92
|
+
- proxy_types
|
93
|
+
- sanger-warren.gemspec
|
94
|
+
homepage: https://github.com/sanger/warren
|
95
|
+
licenses: []
|
96
|
+
metadata:
|
97
|
+
homepage_uri: https://github.com/sanger/warren
|
98
|
+
source_code_uri: https://github.com/sanger/warren
|
99
|
+
changelog_uri: https://github.com/sanger/warren/blob/master/CHANGELOG.md
|
100
|
+
post_install_message:
|
101
|
+
rdoc_options: []
|
102
|
+
require_paths:
|
103
|
+
- lib
|
104
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: 2.6.0
|
109
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - ">="
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '0'
|
114
|
+
requirements: []
|
115
|
+
rubygems_version: 3.1.4
|
116
|
+
signing_key:
|
117
|
+
specification_version: 4
|
118
|
+
summary: Configuring and managing bunny RabbitMQ connections
|
119
|
+
test_files: []
|