pwwka 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +60 -0
- data/README.md +186 -0
- data/Rakefile +15 -0
- data/docs/images/RabbitMQ_Management-2.png +0 -0
- data/docs/images/RabbitMQ_Management-3.png +0 -0
- data/docs/images/RabbitMQ_Management.png +0 -0
- data/lib/pwwka.rb +32 -0
- data/lib/pwwka/channel_connector.rb +31 -0
- data/lib/pwwka/configuration.rb +19 -0
- data/lib/pwwka/handling.rb +15 -0
- data/lib/pwwka/logging.rb +11 -0
- data/lib/pwwka/receiver.rb +77 -0
- data/lib/pwwka/tasks.rb +11 -0
- data/lib/pwwka/test_handler.rb +50 -0
- data/lib/pwwka/transmitter.rb +38 -0
- data/lib/pwwka/version.rb +3 -0
- data/pwwka.gemspec +28 -0
- data/spec/handling_spec.rb +28 -0
- data/spec/logging_spec.rb +19 -0
- data/spec/receiver_spec.rb +100 -0
- data/spec/spec_helper.rb +20 -0
- data/spec/transmitter_spec.rb +72 -0
- metadata +175 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ab413d6cec5031e40f04a92fc87f117bfbab457d
|
4
|
+
data.tar.gz: 0791952942b76c40e8106da29fcbc1ab3051b8d3
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 12554c6f0ed97ba914939cd42f4e37856e9faf7164f74573af3f9f0138cabc7b3122d66827550118d049892067b833af4bf3250697208347e99a21da934d74e9
|
7
|
+
data.tar.gz: e2beebd8979bde2f0846cc24888d6ac1bbf08e287a6aff9b6a40abd5518fa011664df4ccb226298ae9f389c58b8b6e9c5497ad4d53d4ab70e832c64e21b50b47
|
data/.gitignore
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
message_handler
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.1.0
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
pwwka (0.0.2)
|
5
|
+
activemodel
|
6
|
+
activesupport
|
7
|
+
bunny
|
8
|
+
mono_logger
|
9
|
+
sucker_punch
|
10
|
+
|
11
|
+
GEM
|
12
|
+
remote: https://www.rubygems.org/
|
13
|
+
specs:
|
14
|
+
activemodel (4.1.4)
|
15
|
+
activesupport (= 4.1.4)
|
16
|
+
builder (~> 3.1)
|
17
|
+
activesupport (4.1.4)
|
18
|
+
i18n (~> 0.6, >= 0.6.9)
|
19
|
+
json (~> 1.7, >= 1.7.7)
|
20
|
+
minitest (~> 5.1)
|
21
|
+
thread_safe (~> 0.1)
|
22
|
+
tzinfo (~> 1.1)
|
23
|
+
amq-protocol (1.9.2)
|
24
|
+
builder (3.2.2)
|
25
|
+
bunny (1.4.0)
|
26
|
+
amq-protocol (>= 1.9.2)
|
27
|
+
celluloid (0.15.2)
|
28
|
+
timers (~> 1.1.0)
|
29
|
+
diff-lcs (1.2.5)
|
30
|
+
i18n (0.6.11)
|
31
|
+
json (1.8.1)
|
32
|
+
minitest (5.4.0)
|
33
|
+
mono_logger (1.1.0)
|
34
|
+
rake (10.3.2)
|
35
|
+
rspec (3.0.0)
|
36
|
+
rspec-core (~> 3.0.0)
|
37
|
+
rspec-expectations (~> 3.0.0)
|
38
|
+
rspec-mocks (~> 3.0.0)
|
39
|
+
rspec-core (3.0.4)
|
40
|
+
rspec-support (~> 3.0.0)
|
41
|
+
rspec-expectations (3.0.4)
|
42
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
43
|
+
rspec-support (~> 3.0.0)
|
44
|
+
rspec-mocks (3.0.4)
|
45
|
+
rspec-support (~> 3.0.0)
|
46
|
+
rspec-support (3.0.4)
|
47
|
+
sucker_punch (1.1)
|
48
|
+
celluloid (~> 0.15.2)
|
49
|
+
thread_safe (0.3.4)
|
50
|
+
timers (1.1.0)
|
51
|
+
tzinfo (1.2.2)
|
52
|
+
thread_safe (~> 0.1)
|
53
|
+
|
54
|
+
PLATFORMS
|
55
|
+
ruby
|
56
|
+
|
57
|
+
DEPENDENCIES
|
58
|
+
pwwka!
|
59
|
+
rake
|
60
|
+
rspec
|
data/README.md
ADDED
@@ -0,0 +1,186 @@
|
|
1
|
+
This gem connects to a topic exchange on a RabbitMQ server. It gives any app using it the ability to do two things:
|
2
|
+
|
3
|
+
* Transmit messages to the exchange
|
4
|
+
* Receive messages from the exchange and tell the exchange whether or not the message has been acknowledged.
|
5
|
+
|
6
|
+
Any app can do one or both of these things.
|
7
|
+
|
8
|
+
The basic principle of the Pwwka Message Bus is this:
|
9
|
+
|
10
|
+
> The transmitter should send messages to inform anyone who listens that an event has occurred. It's up to the receiver to interpret the message.
|
11
|
+
|
12
|
+
As an example:
|
13
|
+
|
14
|
+
* public_app sends a message that a new client has signed up
|
15
|
+
* admin_app receives that message and updates its client index
|
16
|
+
* email_app receives that message and sends a welcome email to the client
|
17
|
+
|
18
|
+
### Persistence
|
19
|
+
|
20
|
+
All transmitters and receivers share the same exchange. This means that all receivers can read all messages that any transmitter sends. To ensure that all messages are received by eveyone who wants them the Pwwka message bus is configured as follows:
|
21
|
+
|
22
|
+
* The exchange is named and durable. If the service goes down and restarts the named exchange will return with the same settings so everyone can reconnect.
|
23
|
+
* The receiver queues are all named and durable. If the service goes down and restarts the named queue will return with the same settings so everyone can reconnect, and with any unacknowledged messages waiting to be received.
|
24
|
+
* All messages are sent as persistent and require acknowledgement. They will stick around and wait to be received and acknowledged by every queue that wants them, regardless of service interruptions.
|
25
|
+
|
26
|
+
|
27
|
+
## Setting it up
|
28
|
+
|
29
|
+
### Install RabbitMQ locally
|
30
|
+
|
31
|
+
```
|
32
|
+
brew install rabbitmq
|
33
|
+
```
|
34
|
+
|
35
|
+
And follow the instructions.
|
36
|
+
|
37
|
+
### Adding it to your app
|
38
|
+
|
39
|
+
Add to your `Gemfile`:
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
gem 'pwwka', require: false, git: 'https://github.com/stitchfix/pwwka.git'
|
43
|
+
```
|
44
|
+
|
45
|
+
|
46
|
+
### Set up your message_handler configration
|
47
|
+
|
48
|
+
Connect to your RabbitMQ instance using the url and choose a name for your
|
49
|
+
topic exchange.
|
50
|
+
|
51
|
+
In `config/initializers/pwwka`:
|
52
|
+
|
53
|
+
```ruby
|
54
|
+
require 'pwwka'
|
55
|
+
Pwwka.configure do |config|
|
56
|
+
config.rabbit_mq_host = ENV['RABBITMQ_URL']
|
57
|
+
config.topic_exchange_name = "mycompany-topics-#{Rails.env}"
|
58
|
+
end
|
59
|
+
```
|
60
|
+
|
61
|
+
## Sending a message
|
62
|
+
|
63
|
+
You can send any kind of message using `Pwwka::Transmitter.send_message!`:
|
64
|
+
|
65
|
+
```ruby
|
66
|
+
payload = {client_id: '13452564'}
|
67
|
+
routing_key = 'sf.clients.client.created'
|
68
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
69
|
+
```
|
70
|
+
The payload should be a simple hash containing primitives. Don't send objects because the payload will be converted to JSON for sending. This will blow up if an exception is raised. If you want the exception to be rescued and logged, use this instead:
|
71
|
+
|
72
|
+
```ruby
|
73
|
+
Pwwka::Transmitter.send_message_safely(payload, routing_key)
|
74
|
+
```
|
75
|
+
|
76
|
+
You can also use the two convenience methods for sending a message. To include these methods in your class use:
|
77
|
+
|
78
|
+
```ruby
|
79
|
+
include Pwwka::Handling
|
80
|
+
```
|
81
|
+
|
82
|
+
Then you can call:
|
83
|
+
|
84
|
+
```ruby
|
85
|
+
send_message!(payload, routing_key)
|
86
|
+
```
|
87
|
+
|
88
|
+
This method will blow up if something goes wrong. If you want to send safely then use:
|
89
|
+
|
90
|
+
```ruby
|
91
|
+
send_message_safely(payload, routing_key)
|
92
|
+
```
|
93
|
+
|
94
|
+
The messages are not transaction safe so for updates do your best to send them after the transaction commits. You must send create messages after the transaction commits or the receivers will probably not find the persisted records.
|
95
|
+
|
96
|
+
|
97
|
+
## Receiving messages
|
98
|
+
|
99
|
+
The message-handler comes with a rake task you can use in your Procfile to start up your message handler worker:
|
100
|
+
|
101
|
+
```ruby
|
102
|
+
message_handler: rake message_handler:receive HANDLER_KLASS=ClientIndexMessageHandler QUEUE_NAME=adminapp_style_index ROUTING_KEY='client.#.updated'
|
103
|
+
```
|
104
|
+
|
105
|
+
* `HANDLER_KLASS` (required) refers to the class you have to write in you app (equivalent to a `job` in Resque)
|
106
|
+
* `QUEUE_NAME` (required) we must use named queues - see below
|
107
|
+
* `ROUTING_KEY` (optional) defaults to `#.#` (all messages)
|
108
|
+
|
109
|
+
You'll also need to bring the Rake task into your app. For Rails, you'll need to edit the top-level `Rakefile`:
|
110
|
+
|
111
|
+
```ruby
|
112
|
+
require 'stitch_fix/message_handler/tasks'
|
113
|
+
```
|
114
|
+
|
115
|
+
### Queues - what messages will your queue receive
|
116
|
+
|
117
|
+
It depends on your `routing_key`. If you set your routing key to `#.#` (the default) it will receive all the messages. The `#` is a wildcard so if you set it to `client.#` it will receive any message with `client.` at the beginning. The exchange registers the queue's name and routing key so it knows what messages the queue is supposed to receive. A named queue will receive each message it expects to get once and only once.
|
118
|
+
|
119
|
+
The available wildcards are as follows
|
120
|
+
* `*` (star) can substitute for exactly one word.
|
121
|
+
* `#` (hash) can substitute for zero or more words.
|
122
|
+
|
123
|
+
__A note on re-queuing:__ At the moment messages that raise an error on receipt are marked 'not acknowledged, don't resend', and the failure message is logged. All unacknowledged messages will be resent when the worker is restarted. The next iteration of this gem will allow for a specified number of resends without requiring a restart.
|
124
|
+
|
125
|
+
__Spinning up some new dynos to handle the load:__ Since each named queue will receive each message only once you can spin up multiple process using the *same named queue* and they will share the messages between them. If you spin up three processes each will receive roughly one third of the messages, but each message will still only be received once.
|
126
|
+
|
127
|
+
### Handlers
|
128
|
+
Handlers are simple classes that must respond to `self.handle!`. The receiver will send the handler three arguments:
|
129
|
+
|
130
|
+
* `delivery_info` - [a bunch of stuff](http://rubybunny.info/articles/queues.html#accessing_message_delivery_information)
|
131
|
+
* `properties` - [a bunch of other stuff](http://rubybunny.info/articles/queues.html#accessing_message_properties_metadata)
|
132
|
+
* `payload` - the hash sent by the transmitter
|
133
|
+
|
134
|
+
Here is an example:
|
135
|
+
|
136
|
+
```ruby
|
137
|
+
class ClientIndexMessageHandler
|
138
|
+
|
139
|
+
attr_reader :payload
|
140
|
+
def initialize(payload)
|
141
|
+
@payload = payload
|
142
|
+
end
|
143
|
+
|
144
|
+
def self.handle!(delivery_info, properties, payload)
|
145
|
+
# for this handler we only care about the payload
|
146
|
+
handler = new(payload)
|
147
|
+
handler.do_a_thing
|
148
|
+
end
|
149
|
+
|
150
|
+
def do_a_thing
|
151
|
+
###
|
152
|
+
# some stuff that is being done
|
153
|
+
###
|
154
|
+
end
|
155
|
+
|
156
|
+
end
|
157
|
+
```
|
158
|
+
|
159
|
+
## Monitoring
|
160
|
+
RabbitMQ has a good API that should make it easy to set up some simple monitoring. In the meantime there is logging and manual monitoring.
|
161
|
+
|
162
|
+
### Logging
|
163
|
+
The receiver logs details of any exception raised in message handling:
|
164
|
+
```ruby
|
165
|
+
error "Error Processing Message on #{queue_name} -> #{payload}, #{delivery_info.routing_key}: #{e}"
|
166
|
+
```
|
167
|
+
.The transmitter will likewise log an error if you use the `_safely` methods:
|
168
|
+
```ruby
|
169
|
+
error "Error Transmitting Message on #{routing_key} -> #{payload}: #{e}"
|
170
|
+
```
|
171
|
+
|
172
|
+
### Manual monitoring
|
173
|
+
RabbitMQ has a web interface for checking out the health of connections, channels, exchanges and queues. Access it via the Heroku add-ons page for Enigma.
|
174
|
+
|
175
|
+
![RabbitMQ Management 1](docs/images/RabbitMQ_Management.png)
|
176
|
+
![RabbitMQ Management 2](docs/images/RabbitMQ_Management-2.png)
|
177
|
+
![RabbitMQ Management 3](docs/images/RabbitMQ_Management-3.png)
|
178
|
+
|
179
|
+
## Testing
|
180
|
+
The message_handler gem has tests for all its functionality so app testing is best done with expectations. However, if you want to test the message bus end-to-end in your app you can use some helpers in `lib/stitch_fix/message_handler/test_handler.rb`. See the gem specs for examples of how to use them.
|
181
|
+
|
182
|
+
## TODO
|
183
|
+
* automated monitoring
|
184
|
+
* forking
|
185
|
+
* resending
|
186
|
+
* handling messages from inside transactions properly
|
data/Rakefile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'rubygems/package_task'
|
2
|
+
require 'rspec/core/rake_task'
|
3
|
+
require 'bundler'
|
4
|
+
|
5
|
+
$: << File.join(File.dirname(__FILE__),'lib')
|
6
|
+
|
7
|
+
include Rake::DSL
|
8
|
+
|
9
|
+
gemspec = eval(File.read('pwwka.gemspec'))
|
10
|
+
Gem::PackageTask.new(gemspec) {}
|
11
|
+
RSpec::Core::RakeTask.new(:spec)
|
12
|
+
Bundler::GemHelper.install_tasks
|
13
|
+
|
14
|
+
task :default => :spec
|
15
|
+
|
Binary file
|
Binary file
|
Binary file
|
data/lib/pwwka.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
module Pwwka
|
2
|
+
|
3
|
+
class << self
|
4
|
+
def configure
|
5
|
+
yield(configuration)
|
6
|
+
end
|
7
|
+
|
8
|
+
def configuration
|
9
|
+
@configuration ||= Configuration.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def environment
|
13
|
+
ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
require 'json'
|
20
|
+
require 'sucker_punch'
|
21
|
+
require 'active_support/inflector'
|
22
|
+
require 'active_support/core_ext/module'
|
23
|
+
require 'active_support/hash_with_indifferent_access'
|
24
|
+
|
25
|
+
require 'pwwka/version'
|
26
|
+
require 'pwwka/logging'
|
27
|
+
require 'pwwka/channel_connector'
|
28
|
+
require 'pwwka/handling'
|
29
|
+
require 'pwwka/receiver'
|
30
|
+
require 'pwwka/transmitter'
|
31
|
+
|
32
|
+
require 'pwwka/configuration'
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Pwwka
|
2
|
+
class ChannelConnector
|
3
|
+
|
4
|
+
attr_reader :connection
|
5
|
+
attr_reader :topic_exchange_name
|
6
|
+
attr_reader :channel
|
7
|
+
|
8
|
+
# The channel_connector starts the connection to the message_bus
|
9
|
+
# so it should only be instantiated by a method that has a strategy
|
10
|
+
# for closing the connection
|
11
|
+
def initialize
|
12
|
+
configuration = Pwwka.configuration
|
13
|
+
connection_options = {automatically_recover: false}.merge(configuration.options)
|
14
|
+
@connection = Bunny.new(configuration.rabbit_mq_host,
|
15
|
+
connection_options)
|
16
|
+
@topic_exchange_name = configuration.topic_exchange_name
|
17
|
+
@connection.start
|
18
|
+
@channel = @connection.create_channel
|
19
|
+
end
|
20
|
+
|
21
|
+
def topic_exchange
|
22
|
+
@topic_exchange ||= channel.topic(topic_exchange_name, durable: true)
|
23
|
+
end
|
24
|
+
|
25
|
+
def connection_close
|
26
|
+
channel.close
|
27
|
+
connection.close
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'bunny'
|
2
|
+
require 'mono_logger'
|
3
|
+
module Pwwka
|
4
|
+
class Configuration
|
5
|
+
|
6
|
+
attr_accessor :rabbit_mq_host
|
7
|
+
attr_accessor :topic_exchange_name
|
8
|
+
attr_accessor :logger
|
9
|
+
attr_accessor :options
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
@rabbit_mq_host = nil
|
13
|
+
@topic_exchange_name = "pwwka-topics-#{Pwwka.environment}"
|
14
|
+
@logger = MonoLogger.new(STDOUT)
|
15
|
+
@options = {}
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Pwwka
|
2
|
+
|
3
|
+
module Handling
|
4
|
+
|
5
|
+
def send_message!(payload, routing_key)
|
6
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
7
|
+
end
|
8
|
+
|
9
|
+
def send_message_safely(payload, routing_key)
|
10
|
+
Pwwka::Transmitter.send_message_safely(payload, routing_key)
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Pwwka
|
2
|
+
class Receiver
|
3
|
+
|
4
|
+
extend Pwwka::Logging
|
5
|
+
|
6
|
+
attr_reader :channel_connector
|
7
|
+
attr_reader :channel
|
8
|
+
attr_reader :topic_exchange
|
9
|
+
attr_reader :queue_name
|
10
|
+
attr_reader :routing_key
|
11
|
+
|
12
|
+
def initialize(queue_name, routing_key)
|
13
|
+
@queue_name = queue_name
|
14
|
+
@routing_key = routing_key
|
15
|
+
@channel_connector = ChannelConnector.new
|
16
|
+
@channel = @channel_connector.channel
|
17
|
+
@topic_exchange = @channel_connector.topic_exchange
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.subscribe(handler_klass, queue_name, routing_key: "#.#", block: true)
|
21
|
+
raise "#{handler_klass.name} must respond to `handle!`" unless handler_klass.respond_to?(:handle!)
|
22
|
+
receiver = new(queue_name, routing_key)
|
23
|
+
begin
|
24
|
+
info "Receiving on #{queue_name}"
|
25
|
+
receiver.topic_queue.subscribe(ack: true, block: block) do |delivery_info, properties, payload|
|
26
|
+
begin
|
27
|
+
payload = ActiveSupport::HashWithIndifferentAccess.new(JSON.parse(payload))
|
28
|
+
handler_klass.handle!(delivery_info, properties, payload)
|
29
|
+
receiver.ack(delivery_info.delivery_tag)
|
30
|
+
info "Processed Message on #{queue_name} -> #{payload}, #{delivery_info.routing_key}"
|
31
|
+
rescue => e
|
32
|
+
error "Error Processing Message on #{queue_name} -> #{payload}, #{delivery_info.routing_key}: #{e}"
|
33
|
+
# no requeue
|
34
|
+
receiver.nack(delivery_info.delivery_tag)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
rescue Interrupt => _
|
38
|
+
# TODO: trap TERM within channel.work_pool
|
39
|
+
info "Interrupting queue #{queue_name} subscriber safely"
|
40
|
+
receiver.channel_connector.connection_close
|
41
|
+
end
|
42
|
+
return receiver
|
43
|
+
end
|
44
|
+
|
45
|
+
def topic_queue
|
46
|
+
@topic_queue ||= begin
|
47
|
+
queue = channel.queue(queue_name, durable: true)
|
48
|
+
queue.bind(topic_exchange, routing_key: routing_key)
|
49
|
+
queue
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def ack(delivery_tag)
|
54
|
+
channel.acknowledge(delivery_tag, false)
|
55
|
+
end
|
56
|
+
|
57
|
+
def nack(delivery_tag)
|
58
|
+
channel.nack(delivery_tag, false, false)
|
59
|
+
end
|
60
|
+
|
61
|
+
def nack_requeue(delivery_tag)
|
62
|
+
channel.nack(delivery_tag, false, true)
|
63
|
+
end
|
64
|
+
|
65
|
+
def drop_queue
|
66
|
+
topic_queue.purge
|
67
|
+
topic_queue.delete
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_teardown
|
71
|
+
drop_queue
|
72
|
+
topic_exchange.delete
|
73
|
+
channel_connector.connection_close
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
data/lib/pwwka/tasks.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
namespace :message_handler do
|
2
|
+
desc "Start the message bus receiver"
|
3
|
+
task :receive => :environment do
|
4
|
+
raise "HANDLER_KLASS must be set" unless ENV['HANDLER_KLASS']
|
5
|
+
raise "QUEUE_NAME must be set" unless ENV['QUEUE_NAME']
|
6
|
+
handler_klass = ENV['HANDLER_KLASS'].constantize
|
7
|
+
queue_name = "#{ENV['QUEUE_NAME']}_#{Rails.env}"
|
8
|
+
routing_key = ENV['ROUTING_KEY'] || "#.#"
|
9
|
+
Pwwka::Receiver.subscribe(handler_klass, queue_name, routing_key: routing_key)
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module Pwwka
|
2
|
+
class TestHandler
|
3
|
+
|
4
|
+
attr_reader :channel_connector
|
5
|
+
attr_reader :channel
|
6
|
+
attr_reader :topic_exchange
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@channel_connector = ChannelConnector.new
|
10
|
+
@channel = channel_connector.channel
|
11
|
+
@topic_exchange = channel_connector.topic_exchange
|
12
|
+
end
|
13
|
+
|
14
|
+
# call this method to create the queue used for testing
|
15
|
+
# queue needs to be declared before the exchange is published to
|
16
|
+
def test_setup
|
17
|
+
test_queue
|
18
|
+
true
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_queue
|
22
|
+
@test_queue ||= begin
|
23
|
+
test_queue = channel.queue("test-queue", durable: true)
|
24
|
+
test_queue.bind(topic_exchange, routing_key: "*.*")
|
25
|
+
test_queue
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def get_topic_message_payload_for_tests
|
30
|
+
delivery_info, properties, payload = test_queue.pop
|
31
|
+
JSON.parse(payload)
|
32
|
+
end
|
33
|
+
|
34
|
+
def get_topic_message_properties_for_tests
|
35
|
+
delivery_info, properties, payload = test_queue.pop
|
36
|
+
properties
|
37
|
+
end
|
38
|
+
|
39
|
+
def purge_test_queue
|
40
|
+
test_queue.purge
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_teardown
|
44
|
+
test_queue.delete
|
45
|
+
topic_exchange.delete
|
46
|
+
channel_connector.connection_close
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Pwwka
|
2
|
+
class Transmitter
|
3
|
+
|
4
|
+
extend Pwwka::Logging
|
5
|
+
include SuckerPunch::Job
|
6
|
+
|
7
|
+
def self.send_message!(payload, routing_key)
|
8
|
+
new.async.send_message!(payload, routing_key)
|
9
|
+
info "BACKGROUND AFTER Transmitting Message on #{routing_key} -> #{payload}"
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.send_message_safely(payload, routing_key)
|
13
|
+
begin
|
14
|
+
send_message!(payload, routing_key)
|
15
|
+
rescue => e
|
16
|
+
error "Error Transmitting Message on #{routing_key} -> #{payload}: #{e}"
|
17
|
+
return false
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
# send message asynchronously using sucker_punch
|
22
|
+
# call async.send_message!
|
23
|
+
def send_message!(payload, routing_key)
|
24
|
+
self.class.info "BACKGROUND START Transmitting Message on #{routing_key} -> #{payload}"
|
25
|
+
channel_connector = ChannelConnector.new
|
26
|
+
channel_connector.topic_exchange.publish(
|
27
|
+
payload.to_json,
|
28
|
+
routing_key: routing_key,
|
29
|
+
persistent: true)
|
30
|
+
channel_connector.connection_close
|
31
|
+
# if it gets this far it has succeeded
|
32
|
+
self.class.info "BACKGROUND END Transmitting Message on #{routing_key} -> #{payload}"
|
33
|
+
return true
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
data/pwwka.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require 'pwwka/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "pwwka"
|
7
|
+
s.version = Pwwka::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ['Stitch Fix Engineering']
|
10
|
+
s.email = ['eng@stitchfix.com']
|
11
|
+
s.homepage = "http://www.stitchfix.com"
|
12
|
+
s.summary = "Send and receive messages via RabbitMQ"
|
13
|
+
s.description = "The purpose of this gem is to normalise the sending and
|
14
|
+
receiving of messages between Rails apps using the shared RabbitMQ
|
15
|
+
message bus"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
s.add_dependency("bunny")
|
22
|
+
s.add_dependency("activesupport")
|
23
|
+
s.add_dependency("activemodel")
|
24
|
+
s.add_dependency("sucker_punch")
|
25
|
+
s.add_dependency("mono_logger")
|
26
|
+
s.add_development_dependency("rake")
|
27
|
+
s.add_development_dependency("rspec")
|
28
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Pwwka::Handling do
|
4
|
+
|
5
|
+
class HKlass
|
6
|
+
include Pwwka::Handling
|
7
|
+
end
|
8
|
+
|
9
|
+
describe "adding handler methods" do
|
10
|
+
|
11
|
+
let(:handling_class) { HKlass.new }
|
12
|
+
let(:payload) { { this: 'that'} }
|
13
|
+
let(:routing_key) { 'sf.merch.style.updated' }
|
14
|
+
|
15
|
+
it "should respond to 'send_message!'" do
|
16
|
+
expect(Pwwka::Transmitter).to receive(:send_message!).with(payload, routing_key)
|
17
|
+
handling_class.send_message!(payload, routing_key)
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should respond to 'send_message_safely'" do
|
21
|
+
expect(Pwwka::Transmitter).to receive(:send_message_safely).with(payload, routing_key)
|
22
|
+
handling_class.send_message_safely(payload, routing_key)
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'spec_helper.rb'
|
2
|
+
|
3
|
+
describe Pwwka::Logging do
|
4
|
+
|
5
|
+
class ForLogging
|
6
|
+
extend Pwwka::Logging
|
7
|
+
end
|
8
|
+
|
9
|
+
it "returns the logger" do
|
10
|
+
expect(ForLogging.logger).to be_instance_of(MonoLogger)
|
11
|
+
end
|
12
|
+
|
13
|
+
%w(debug info error fatal).each do |severity|
|
14
|
+
it "logs #{severity} messages at the class level" do
|
15
|
+
expect(ForLogging.respond_to?(severity.to_sym)).to eq true
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,100 @@
|
|
1
|
+
require 'spec_helper.rb'
|
2
|
+
|
3
|
+
describe Pwwka::Receiver do
|
4
|
+
|
5
|
+
class HandyHandler
|
6
|
+
def self.handle!(delivery_info, properties, payload)
|
7
|
+
return "made it here"
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
let(:payload) { Hash[:this, "that"] }
|
12
|
+
let(:routing_key) { "this.that" }
|
13
|
+
let(:queue_name) { "receiver_test" }
|
14
|
+
|
15
|
+
describe "::subscribe" do
|
16
|
+
|
17
|
+
before(:each) do
|
18
|
+
@receiver = Pwwka::Receiver.subscribe(HandyHandler, "receiver_test", block: false)
|
19
|
+
end
|
20
|
+
|
21
|
+
after(:each) do
|
22
|
+
@receiver.test_teardown
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should receive the sent message" do
|
26
|
+
expect(HandyHandler).to receive(:handle!).and_return("made it here")
|
27
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should nack the sent message if an error is raised" do
|
31
|
+
expect(HandyHandler).to receive(:handle!).and_raise("blow up")
|
32
|
+
expect(@receiver).not_to receive(:ack)
|
33
|
+
expect(@receiver).to receive(:nack).with(instance_of(Fixnum))
|
34
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
describe "instance methods and ::new" do
|
40
|
+
|
41
|
+
before(:each) do
|
42
|
+
@receiver = Pwwka::Receiver.new(queue_name, routing_key)
|
43
|
+
end
|
44
|
+
|
45
|
+
after(:each) do
|
46
|
+
@receiver.test_teardown
|
47
|
+
end
|
48
|
+
|
49
|
+
describe "::new" do
|
50
|
+
|
51
|
+
it "should initialize the expected attributes" do
|
52
|
+
expect(@receiver.topic_exchange.name).to eq("topics-test")
|
53
|
+
expect(@receiver.topic_exchange.type).to eq(:topic)
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
describe "#topic_queue" do
|
59
|
+
|
60
|
+
it "should return the queue with the right attributes" do
|
61
|
+
queue = @receiver.topic_queue
|
62
|
+
expect(queue.name).to eq(queue_name)
|
63
|
+
expect(queue.instance_variable_get(:@bindings).count).to eq(1)
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
describe "#ack" do
|
69
|
+
|
70
|
+
it "should call the correct channel method" do
|
71
|
+
delivery_tag = 1224
|
72
|
+
expect(@receiver.channel).to receive(:acknowledge).with(delivery_tag, false)
|
73
|
+
@receiver.ack(delivery_tag)
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
describe "#nack" do
|
79
|
+
|
80
|
+
it "should call the correct channel method" do
|
81
|
+
delivery_tag = 1224
|
82
|
+
expect(@receiver.channel).to receive(:nack).with(delivery_tag, false, false)
|
83
|
+
@receiver.nack(delivery_tag)
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
|
88
|
+
describe "#nack_requeue" do
|
89
|
+
|
90
|
+
it "should call the correct channel method" do
|
91
|
+
delivery_tag = 1224
|
92
|
+
expect(@receiver.channel).to receive(:nack).with(delivery_tag, false, true)
|
93
|
+
@receiver.nack_requeue(delivery_tag)
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
GEM_ROOT = File.expand_path(File.join(File.dirname(__FILE__),'..'))
|
2
|
+
require 'pwwka'
|
3
|
+
require 'pwwka/test_handler'
|
4
|
+
require 'sucker_punch/testing/inline'
|
5
|
+
Dir["#{GEM_ROOT}/spec/support/**/*.rb"].sort.each {|f| require f}
|
6
|
+
|
7
|
+
RSpec.configure do |config|
|
8
|
+
|
9
|
+
SuckerPunch.logger = nil
|
10
|
+
|
11
|
+
config.expect_with :rspec do |c|
|
12
|
+
c.syntax = :expect
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
Pwwka.configure do |config|
|
18
|
+
config.topic_exchange_name = "topics-test"
|
19
|
+
config.logger = MonoLogger.new("/dev/null")
|
20
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'spec_helper.rb'
|
2
|
+
|
3
|
+
describe Pwwka::Transmitter do
|
4
|
+
|
5
|
+
before(:all) do
|
6
|
+
@test_handler = Pwwka::TestHandler.new
|
7
|
+
@test_handler.test_setup
|
8
|
+
end
|
9
|
+
|
10
|
+
after(:all) { @test_handler.test_teardown }
|
11
|
+
|
12
|
+
let(:payload) { Hash[:this, "that"] }
|
13
|
+
let(:routing_key) { "this.that" }
|
14
|
+
|
15
|
+
describe "#send_message!" do
|
16
|
+
|
17
|
+
it "should send the correct payload" do
|
18
|
+
success = Pwwka::Transmitter.new.async.send_message!(payload, routing_key)
|
19
|
+
expect(success).to be_truthy
|
20
|
+
received_payload = @test_handler.get_topic_message_payload_for_tests
|
21
|
+
expect(received_payload["this"]).to eq("that")
|
22
|
+
end
|
23
|
+
|
24
|
+
it "should blow up if exception raised" do
|
25
|
+
expect(Pwwka::ChannelConnector).to receive(:new).and_raise("blow up")
|
26
|
+
expect {
|
27
|
+
Pwwka::Transmitter.new.async.send_message!(payload, routing_key)
|
28
|
+
}.to raise_error
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
describe "::send_message!" do
|
34
|
+
|
35
|
+
it "should send the correct payload" do
|
36
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
37
|
+
received_payload = @test_handler.get_topic_message_payload_for_tests
|
38
|
+
expect(received_payload["this"]).to eq("that")
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should use sucker_punch to send the message in the background" do
|
42
|
+
expect_any_instance_of(Pwwka::Transmitter).to receive(:send_message!).with(payload, routing_key).and_return(true)
|
43
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should blow up if exception raised" do
|
47
|
+
expect(Pwwka::ChannelConnector).to receive(:new).and_raise("blow up")
|
48
|
+
expect{
|
49
|
+
Pwwka::Transmitter.send_message!(payload, routing_key)
|
50
|
+
}.to raise_error
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
describe "::send_message_safely" do
|
56
|
+
|
57
|
+
it "should send the correct payload" do
|
58
|
+
Pwwka::Transmitter.send_message_safely(payload, routing_key)
|
59
|
+
received_payload = @test_handler.get_topic_message_payload_for_tests
|
60
|
+
expect(received_payload["this"]).to eq("that")
|
61
|
+
end
|
62
|
+
|
63
|
+
it "should not blow up if exception raised" do
|
64
|
+
expect(Pwwka::ChannelConnector).to receive(:new).and_raise("blow up")
|
65
|
+
Pwwka::Transmitter.send_message_safely(payload, routing_key)
|
66
|
+
# check nothing has been queued
|
67
|
+
expect(@test_handler.test_queue.pop.compact.count).to eq(0)
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
metadata
ADDED
@@ -0,0 +1,175 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: pwwka
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Stitch Fix Engineering
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-08-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bunny
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activesupport
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: activemodel
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: sucker_punch
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: mono_logger
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rake
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
description: |-
|
112
|
+
The purpose of this gem is to normalise the sending and
|
113
|
+
receiving of messages between Rails apps using the shared RabbitMQ
|
114
|
+
message bus
|
115
|
+
email:
|
116
|
+
- eng@stitchfix.com
|
117
|
+
executables: []
|
118
|
+
extensions: []
|
119
|
+
extra_rdoc_files: []
|
120
|
+
files:
|
121
|
+
- ".gitignore"
|
122
|
+
- ".ruby-gemset"
|
123
|
+
- ".ruby-version"
|
124
|
+
- Gemfile
|
125
|
+
- Gemfile.lock
|
126
|
+
- README.md
|
127
|
+
- Rakefile
|
128
|
+
- docs/images/RabbitMQ_Management-2.png
|
129
|
+
- docs/images/RabbitMQ_Management-3.png
|
130
|
+
- docs/images/RabbitMQ_Management.png
|
131
|
+
- lib/pwwka.rb
|
132
|
+
- lib/pwwka/channel_connector.rb
|
133
|
+
- lib/pwwka/configuration.rb
|
134
|
+
- lib/pwwka/handling.rb
|
135
|
+
- lib/pwwka/logging.rb
|
136
|
+
- lib/pwwka/receiver.rb
|
137
|
+
- lib/pwwka/tasks.rb
|
138
|
+
- lib/pwwka/test_handler.rb
|
139
|
+
- lib/pwwka/transmitter.rb
|
140
|
+
- lib/pwwka/version.rb
|
141
|
+
- pwwka.gemspec
|
142
|
+
- spec/handling_spec.rb
|
143
|
+
- spec/logging_spec.rb
|
144
|
+
- spec/receiver_spec.rb
|
145
|
+
- spec/spec_helper.rb
|
146
|
+
- spec/transmitter_spec.rb
|
147
|
+
homepage: http://www.stitchfix.com
|
148
|
+
licenses: []
|
149
|
+
metadata: {}
|
150
|
+
post_install_message:
|
151
|
+
rdoc_options: []
|
152
|
+
require_paths:
|
153
|
+
- lib
|
154
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - ">="
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
|
+
requirements:
|
161
|
+
- - ">="
|
162
|
+
- !ruby/object:Gem::Version
|
163
|
+
version: '0'
|
164
|
+
requirements: []
|
165
|
+
rubyforge_project:
|
166
|
+
rubygems_version: 2.2.0
|
167
|
+
signing_key:
|
168
|
+
specification_version: 4
|
169
|
+
summary: Send and receive messages via RabbitMQ
|
170
|
+
test_files:
|
171
|
+
- spec/handling_spec.rb
|
172
|
+
- spec/logging_spec.rb
|
173
|
+
- spec/receiver_spec.rb
|
174
|
+
- spec/spec_helper.rb
|
175
|
+
- spec/transmitter_spec.rb
|