emque-consuming 1.0.0.beta4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +40 -0
- data/.travis.yml +10 -0
- data/CHANGELOG.md +11 -0
- data/Gemfile +7 -0
- data/LICENSE.txt +22 -0
- data/README.md +205 -0
- data/Rakefile +14 -0
- data/bin/emque +5 -0
- data/emque-consuming.gemspec +36 -0
- data/lib/emque/consuming/actor.rb +21 -0
- data/lib/emque/consuming/adapter.rb +47 -0
- data/lib/emque/consuming/adapters/rabbit_mq/manager.rb +111 -0
- data/lib/emque/consuming/adapters/rabbit_mq/retry_worker.rb +59 -0
- data/lib/emque/consuming/adapters/rabbit_mq/worker.rb +87 -0
- data/lib/emque/consuming/adapters/rabbit_mq.rb +26 -0
- data/lib/emque/consuming/application.rb +112 -0
- data/lib/emque/consuming/cli.rb +140 -0
- data/lib/emque/consuming/command_receivers/base.rb +37 -0
- data/lib/emque/consuming/command_receivers/http_server.rb +103 -0
- data/lib/emque/consuming/command_receivers/unix_socket.rb +169 -0
- data/lib/emque/consuming/configuration.rb +63 -0
- data/lib/emque/consuming/consumer/common.rb +61 -0
- data/lib/emque/consuming/consumer.rb +33 -0
- data/lib/emque/consuming/consuming.rb +27 -0
- data/lib/emque/consuming/control/errors.rb +41 -0
- data/lib/emque/consuming/control/workers.rb +23 -0
- data/lib/emque/consuming/control.rb +33 -0
- data/lib/emque/consuming/core.rb +89 -0
- data/lib/emque/consuming/error_tracker.rb +39 -0
- data/lib/emque/consuming/generators/application.rb +95 -0
- data/lib/emque/consuming/helpers.rb +29 -0
- data/lib/emque/consuming/logging.rb +32 -0
- data/lib/emque/consuming/message.rb +22 -0
- data/lib/emque/consuming/pidfile.rb +54 -0
- data/lib/emque/consuming/router.rb +73 -0
- data/lib/emque/consuming/runner.rb +168 -0
- data/lib/emque/consuming/status.rb +26 -0
- data/lib/emque/consuming/tasks.rb +121 -0
- data/lib/emque/consuming/transmitter.rb +31 -0
- data/lib/emque/consuming/version.rb +5 -0
- data/lib/emque/consuming.rb +9 -0
- data/lib/emque-consuming.rb +3 -0
- data/lib/templates/.gitignore.tt +25 -0
- data/lib/templates/Gemfile.tt +6 -0
- data/lib/templates/Rakefile.tt +7 -0
- data/lib/templates/config/application.rb.tt +42 -0
- data/lib/templates/config/environments/development.rb.tt +2 -0
- data/lib/templates/config/environments/production.rb.tt +2 -0
- data/lib/templates/config/environments/staging.rb.tt +2 -0
- data/lib/templates/config/environments/test.rb.tt +2 -0
- data/lib/templates/config/routes.rb.tt +8 -0
- data/spec/application_spec.rb +28 -0
- data/spec/cli_spec.rb +136 -0
- data/spec/configuration_spec.rb +47 -0
- data/spec/consumer_spec.rb +56 -0
- data/spec/control/errors_spec.rb +170 -0
- data/spec/control_spec.rb +15 -0
- data/spec/core_spec.rb +121 -0
- data/spec/dummy/config/application.rb +38 -0
- data/spec/dummy/config/environments/test.rb +0 -0
- data/spec/dummy/config/routes.rb +0 -0
- data/spec/error_tracker_spec.rb +64 -0
- data/spec/pidfile_spec.rb +74 -0
- data/spec/router_spec.rb +14 -0
- data/spec/runner_spec.rb +138 -0
- data/spec/spec_helper.rb +43 -0
- metadata +309 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 40887d498ed3a5b0237e45e3356b28c8b5b4d2df
|
4
|
+
data.tar.gz: 6d19d4ff37dbaf233dfe696e7bf03a5650a5a986
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5de0803235bb52e85a39d61a944cb0ce60df327f1f9b51a855214cadc2041c5c4bb7825126cb95a01619a0911b865de34b820aa26ec13a640d12100bbe9de1d1
|
7
|
+
data.tar.gz: b6e8634ba543e1b9f3bcd9b859219b601671413a16341beb2a35d184909c4125122cccca842ce24554a8d0614ea2910cff251975bcd9eade678ff371446721d4
|
data/.gitignore
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
kafka_*/
|
12
|
+
/log/*
|
13
|
+
/logs/*
|
14
|
+
log-cleaner.log
|
15
|
+
Gemfile.lock
|
16
|
+
*.log
|
17
|
+
|
18
|
+
## Specific to RubyMotion:
|
19
|
+
.dat*
|
20
|
+
.repl_history
|
21
|
+
build/
|
22
|
+
|
23
|
+
## Documentation cache and generated files:
|
24
|
+
/.yardoc/
|
25
|
+
/_yardoc/
|
26
|
+
/doc/
|
27
|
+
/rdoc/
|
28
|
+
|
29
|
+
## Environment normalisation:
|
30
|
+
/.bundle/
|
31
|
+
/lib/bundler/man/
|
32
|
+
|
33
|
+
# for a library or gem, you might want to ignore these files since the code is
|
34
|
+
# intended to run in multiple environments; otherwise, check them in:
|
35
|
+
# Gemfile.lock
|
36
|
+
# .ruby-version
|
37
|
+
# .ruby-gemset
|
38
|
+
|
39
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
40
|
+
.rvmrc
|
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
## 1.0.0.beta4 — (unreleased)
|
2
|
+
|
3
|
+
### BREAKING CHANGE - New Queue Names
|
4
|
+
Applications updating to this version will have new queue names in RabbitMQ.
|
5
|
+
After starting up, messages will need to be manually moved
|
6
|
+
from the old queue to the new one.
|
7
|
+
|
8
|
+
### Failed Message Routing
|
9
|
+
Messages that are not acknolwedged due to a consumer error will now be routed
|
10
|
+
into a `service_name.error` queue. These can then be inspected and be discarded,
|
11
|
+
purged, or manually moved back to the primary queue for re-processing.
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 TeamSnap
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,205 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/emque/emque-consuming.png)](https://travis-ci.org/emque/emque-consuming)
|
2
|
+
|
3
|
+
# Emque::Consuming
|
4
|
+
|
5
|
+
Emque Consuming is a Ruby application framework that includes everything needed
|
6
|
+
to create and run application capable of consuming messages from a message
|
7
|
+
broker in a Pub/sub architecture. Messages can be produced with the
|
8
|
+
[emque-producing](https://github.com/teamsnap/emque-producing) library.
|
9
|
+
|
10
|
+
## Adapters
|
11
|
+
|
12
|
+
We currently only support RabbitMQ. If you would like to add your own adapter,
|
13
|
+
take a look at [the adapters directory](https://github.com/teamsnap/emque-consuming/tree/socket-control/lib/emque/consuming/adapters).
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
Add this line to your application's Gemfile:
|
18
|
+
|
19
|
+
```ruby
|
20
|
+
gem "emque-consuming", :git => "https://github.com/teamsnap/emque-consuming"
|
21
|
+
# make sure you have bunny for rabbitmq unless you're using a custom adapter
|
22
|
+
gem "bunny", "~> 1.7"
|
23
|
+
|
24
|
+
```
|
25
|
+
|
26
|
+
And then execute:
|
27
|
+
|
28
|
+
$ bundle
|
29
|
+
|
30
|
+
Or install it yourself as:
|
31
|
+
|
32
|
+
$ gem install emque-consuming
|
33
|
+
|
34
|
+
## Setup
|
35
|
+
|
36
|
+
### Easy
|
37
|
+
|
38
|
+
Using the `new` generator is the easiest way to get up and running quickly.
|
39
|
+
|
40
|
+
```
|
41
|
+
emque <options> new name
|
42
|
+
```
|
43
|
+
|
44
|
+
This command will create a directory "name" with barebones directories and files
|
45
|
+
you'll need to get started. You can also use the following command line options
|
46
|
+
to customize your application's configuration. Options will be added to the
|
47
|
+
config/application.rb file generated.
|
48
|
+
|
49
|
+
```
|
50
|
+
emque <options> (start|stop|new|console|help) <name (new only)>
|
51
|
+
# ...
|
52
|
+
-S, --socket PATH PATH to the application's unix socket
|
53
|
+
-b, --bind IP:PORT IP & port for the http status application to listen on.
|
54
|
+
# ...
|
55
|
+
-e, --error-limit N Set the max errors before application suicide
|
56
|
+
-s, --status Run the http status application
|
57
|
+
-x, --error-expiration SECONDS Expire errors after SECONDS
|
58
|
+
--app-name NAME Run the application as NAME
|
59
|
+
# ...
|
60
|
+
```
|
61
|
+
|
62
|
+
### Custom
|
63
|
+
|
64
|
+
Configure Emque::Consuming in your config/application.rb file. Here is an example:
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
# config/application.rb
|
68
|
+
require "emque/consuming"
|
69
|
+
|
70
|
+
module Example
|
71
|
+
class Application
|
72
|
+
include Emque::Consuming::Application
|
73
|
+
|
74
|
+
initialize_core!
|
75
|
+
|
76
|
+
config.set_adapter(:rabbit_mq, :url => ENV["RABBITMQ_URL"], :prefetch => 10)
|
77
|
+
|
78
|
+
# customize the error thresholds
|
79
|
+
# config.error_limit = 10
|
80
|
+
# config.error_expiration = 300
|
81
|
+
|
82
|
+
# enable the status application
|
83
|
+
# config.status = :on
|
84
|
+
|
85
|
+
# errors will be logged, but if more is needed, that can be added
|
86
|
+
# config.error_handlers << Proc.new {|ex, context|
|
87
|
+
# send an email, send to HoneyBadger, etc
|
88
|
+
# }
|
89
|
+
|
90
|
+
# do something when shutdown occurs
|
91
|
+
# config.shutdown_handlers << Proc.new { |context|
|
92
|
+
# notify slack, pagerduty or send an email
|
93
|
+
# }
|
94
|
+
end
|
95
|
+
end
|
96
|
+
```
|
97
|
+
|
98
|
+
You'll also want to set up a routes.rb file:
|
99
|
+
|
100
|
+
```ruby
|
101
|
+
# config/routes.rb
|
102
|
+
|
103
|
+
Example::Application.router.map do
|
104
|
+
topic "events" => EventsConsumer do
|
105
|
+
map "events.new" => "new_event"
|
106
|
+
# ...
|
107
|
+
end
|
108
|
+
|
109
|
+
# ...
|
110
|
+
end
|
111
|
+
```
|
112
|
+
|
113
|
+
and a consumer for each topic:
|
114
|
+
|
115
|
+
```ruby
|
116
|
+
# app/consumers/events_consumer.rb
|
117
|
+
|
118
|
+
class EventsConsumer
|
119
|
+
include Emque::Consuming.consumer
|
120
|
+
|
121
|
+
def new_event(message)
|
122
|
+
# NOTE: message is an immutable [Virtus](https://github.com/solnic/virtus) Value Object.
|
123
|
+
# Check it out here: https://github.com/teamsnap/emque-consuming/blob/master/lib/emque/consuming/message.rb
|
124
|
+
|
125
|
+
# You don't have to use (pipe)[https://github.com/teamsnap/emque-consuming/blob/master/lib/emque/consuming/consumer/common.rb#L23], be we love it!
|
126
|
+
pipe(message, :through => [
|
127
|
+
:shared_action, :do_something_with_new_event
|
128
|
+
])
|
129
|
+
end
|
130
|
+
|
131
|
+
private
|
132
|
+
|
133
|
+
def shared_action(message)
|
134
|
+
# ...
|
135
|
+
end
|
136
|
+
|
137
|
+
def do_something_with_new_event(message)
|
138
|
+
# ...
|
139
|
+
end
|
140
|
+
end
|
141
|
+
```
|
142
|
+
|
143
|
+
## Usage
|
144
|
+
|
145
|
+
Emque::Consuming provides a command line interface:
|
146
|
+
|
147
|
+
```
|
148
|
+
$ bundle exec emque help
|
149
|
+
|
150
|
+
emque <options> (start|stop|new|console|help) <name (new only)>
|
151
|
+
-P, --pidfile PATH Store pid in PATH
|
152
|
+
-S, --socket PATH PATH to the application's unix socket
|
153
|
+
-b, --bind IP:PORT IP & port for the http status application to listen on.
|
154
|
+
-d, --daemon Daemonize the application
|
155
|
+
-e, --error-limit N Set the max errors before application suicide
|
156
|
+
-s, --status Run the http status application
|
157
|
+
-x, --error-expiration SECONDS Expire errors after SECONDS
|
158
|
+
--app-name NAME Run the application as NAME
|
159
|
+
--env (ex. production) Set the application environment, overrides EMQUE_ENV
|
160
|
+
```
|
161
|
+
|
162
|
+
and a series of rake commands:
|
163
|
+
|
164
|
+
```
|
165
|
+
$ bundle exec rake -T
|
166
|
+
|
167
|
+
rake emque:configuration # Show the current configuration of a running instance (accepts SOCKET)
|
168
|
+
rake emque:console # Start a pry console
|
169
|
+
rake emque:errors:clear # Clear all outstanding errors (accepts SOCKET)
|
170
|
+
rake emque:errors:expire_after # Change the number of seconds to SECONDS before future errors expire (accepts SOCKET)
|
171
|
+
rake emque:errors:limit:down # Decrease the error limit (accepts SOCKET)
|
172
|
+
rake emque:errors:limit:up # Increase the error limit (accepts SOCKET)
|
173
|
+
rake emque:restart # Restart the workers inside a running instance (does not reload code; accepts SOCKET)
|
174
|
+
rake emque:routes # Show the available routes
|
175
|
+
rake emque:start # Start a new instance (accepts PIDFILE, DAEMON)
|
176
|
+
rake emque:status # Show the current status of a running instance (accepts SOCKET)
|
177
|
+
rake emque:stop # Stop a running instance (accepts SOCKET)
|
178
|
+
```
|
179
|
+
|
180
|
+
To use the rake commands, add the following lines to your application's Rakefile:
|
181
|
+
|
182
|
+
```ruby
|
183
|
+
require_relative "config/application"
|
184
|
+
require "emque/consuming/tasks"
|
185
|
+
```
|
186
|
+
|
187
|
+
## Tests
|
188
|
+
|
189
|
+
Testing is a bit sparse at the moment, but we're working on it.
|
190
|
+
|
191
|
+
To run tests...
|
192
|
+
|
193
|
+
```
|
194
|
+
bundle exec rspec
|
195
|
+
```
|
196
|
+
|
197
|
+
## Contributing
|
198
|
+
|
199
|
+
FIRST: Read our style guides at https://github.com/teamsnap/guides/tree/master/ruby
|
200
|
+
|
201
|
+
1. Fork it ( http://github.com/teamsnap/emque-consuming/fork )
|
202
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
203
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
204
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
205
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require "rake/clean"
|
4
|
+
require "rspec/core/rake_task"
|
5
|
+
require "coveralls/rake/task"
|
6
|
+
|
7
|
+
CLOBBER.include("coverage")
|
8
|
+
|
9
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
10
|
+
t.fail_on_error = false
|
11
|
+
end
|
12
|
+
task :default => :spec
|
13
|
+
|
14
|
+
Coveralls::RakeTask.new
|
data/bin/emque
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "emque/consuming/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "emque-consuming"
|
8
|
+
spec.version = Emque::Consuming::VERSION
|
9
|
+
spec.authors = ["Ryan Williams", "Dan Matthews"]
|
10
|
+
spec.email = ["oss@teamsnap.com"]
|
11
|
+
spec.summary = %q{Microservices framework for Ruby}
|
12
|
+
spec.summary = %q{Microservices framework for Ruby}
|
13
|
+
spec.homepage = "https://github.com/teamsnap/emque-consuming"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
spec.required_ruby_version = "~> 2.1.2"
|
21
|
+
|
22
|
+
spec.add_dependency "celluloid", "0.16.0"
|
23
|
+
spec.add_dependency "dante", "~> 0.2.0"
|
24
|
+
spec.add_dependency "oj", "~> 2.11.4"
|
25
|
+
spec.add_dependency "virtus", "~> 1.0"
|
26
|
+
spec.add_dependency "puma", "~> 2.11.1"
|
27
|
+
spec.add_dependency "pipe-ruby", "~> 0.2.0"
|
28
|
+
spec.add_dependency "inflecto", "~> 0.0.2"
|
29
|
+
|
30
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
31
|
+
spec.add_development_dependency "rake", "~> 10.4.2"
|
32
|
+
spec.add_development_dependency "rspec", "~> 3.3"
|
33
|
+
spec.add_development_dependency "bunny", "~> 1.7"
|
34
|
+
spec.add_development_dependency "timecop", "~> 0.7.1"
|
35
|
+
spec.add_development_dependency "daemon_controller", "~> 1.2.0"
|
36
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Emque
|
2
|
+
module Consuming
|
3
|
+
module Actor
|
4
|
+
def self.included(descendant)
|
5
|
+
descendant.class_eval do
|
6
|
+
include Celluloid
|
7
|
+
include Emque::Consuming::Helpers
|
8
|
+
attr_accessor :shutdown
|
9
|
+
private :shutdown=
|
10
|
+
private :shutdown
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def stop(&block)
|
15
|
+
self.shutdown = true
|
16
|
+
block.call if block_given?
|
17
|
+
terminate
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require "inflecto"
|
2
|
+
|
3
|
+
module Emque
|
4
|
+
module Consuming
|
5
|
+
class AdapterConfigurationError < StandardError; end
|
6
|
+
|
7
|
+
module Adapters; end
|
8
|
+
|
9
|
+
class Adapter
|
10
|
+
extend Forwardable
|
11
|
+
|
12
|
+
attr_reader :name, :options
|
13
|
+
|
14
|
+
def_delegators :namespace, :default_options, :manager
|
15
|
+
|
16
|
+
def initialize(name, opts = {})
|
17
|
+
self.name = name
|
18
|
+
fetch_and_load
|
19
|
+
self.options = default_options.merge(opts)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
attr_writer :name, :options
|
25
|
+
attr_accessor :namespace
|
26
|
+
|
27
|
+
def fetch_and_load
|
28
|
+
klass = Inflecto.camelize(name.to_s)
|
29
|
+
|
30
|
+
unless Emque::Consuming::Adapters.const_defined?(klass)
|
31
|
+
require "emque/consuming/adapters/#{name}"
|
32
|
+
end
|
33
|
+
|
34
|
+
self.namespace = Inflecto.constantize(
|
35
|
+
"Emque::Consuming::Adapters::#{klass}"
|
36
|
+
)
|
37
|
+
|
38
|
+
namespace.load
|
39
|
+
rescue LoadError
|
40
|
+
raise AdapterConfigurationError, "Unable to load requested adapter"
|
41
|
+
rescue NameError => e
|
42
|
+
$stdout.puts e.inspect
|
43
|
+
raise AdapterConfigurationError, "Unknown consuming adapter"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require "bunny"
|
2
|
+
|
3
|
+
module Emque
|
4
|
+
module Consuming
|
5
|
+
module Adapters
|
6
|
+
module RabbitMq
|
7
|
+
class Manager
|
8
|
+
include Emque::Consuming::Actor
|
9
|
+
trap_exit :actor_died
|
10
|
+
|
11
|
+
def actor_died(actor, reason)
|
12
|
+
unless shutdown
|
13
|
+
logger.error "RabbitMQ Manager: actor_died - #{actor.inspect} "+
|
14
|
+
"died: #{reason}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def start
|
19
|
+
setup_connection
|
20
|
+
initialize_error_queue
|
21
|
+
initialize_workers
|
22
|
+
logger.info "RabbitMQ Manager: starting #{worker_count} workers..."
|
23
|
+
workers(:flatten => true).each do |worker|
|
24
|
+
worker.async.start
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def stop
|
29
|
+
logger.info "RabbitMQ Manager: stopping #{worker_count} workers..."
|
30
|
+
|
31
|
+
super do
|
32
|
+
workers(:flatten => true).each do |worker|
|
33
|
+
logger.info "RabbitMQ Manager: stopping #{worker.topic} worker..."
|
34
|
+
worker.stop
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
@connection.stop
|
39
|
+
end
|
40
|
+
|
41
|
+
def worker(topic:, command:)
|
42
|
+
if workers.has_key?(topic)
|
43
|
+
case command
|
44
|
+
when :down
|
45
|
+
worker = workers[topic].pop
|
46
|
+
worker.stop if worker
|
47
|
+
when :up
|
48
|
+
workers[topic] << new_worker(topic)
|
49
|
+
workers[topic].last.async.start
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def workers(flatten: false)
|
55
|
+
flatten ? @workers.values.flatten : @workers
|
56
|
+
end
|
57
|
+
|
58
|
+
def retry_errors
|
59
|
+
RetryWorker.new(@connection).retry_errors
|
60
|
+
end
|
61
|
+
|
62
|
+
private
|
63
|
+
|
64
|
+
attr_writer :workers
|
65
|
+
|
66
|
+
def initialize_workers
|
67
|
+
self.workers = {}.tap { |workers|
|
68
|
+
router.topic_mapping.keys.each do |topic|
|
69
|
+
workers[topic] ||= []
|
70
|
+
router.workers(topic).times do
|
71
|
+
workers[topic] << new_worker(topic)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
}
|
75
|
+
end
|
76
|
+
|
77
|
+
def initialize_error_queue
|
78
|
+
channel = @connection.create_channel
|
79
|
+
error_exchange = channel.fanout(
|
80
|
+
"#{config.app_name}.error",
|
81
|
+
:durable => true,
|
82
|
+
:auto_delete => false
|
83
|
+
)
|
84
|
+
channel.queue(
|
85
|
+
"emque.#{config.app_name}.error",
|
86
|
+
:durable => true,
|
87
|
+
:auto_delete => false,
|
88
|
+
:arguments => {
|
89
|
+
"x-dead-letter-exchange" => "#{config.app_name}.error"
|
90
|
+
}
|
91
|
+
).bind(error_exchange)
|
92
|
+
channel.close
|
93
|
+
end
|
94
|
+
|
95
|
+
def new_worker(topic)
|
96
|
+
Worker.new_link(@connection, topic)
|
97
|
+
end
|
98
|
+
|
99
|
+
def setup_connection
|
100
|
+
@connection = Bunny.new(config.adapter.options[:url])
|
101
|
+
@connection.start
|
102
|
+
end
|
103
|
+
|
104
|
+
def worker_count
|
105
|
+
workers(:flatten => true).size
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
require "bunny"
|
2
|
+
|
3
|
+
module Emque
|
4
|
+
module Consuming
|
5
|
+
module Adapters
|
6
|
+
module RabbitMq
|
7
|
+
class RetryWorker
|
8
|
+
include Emque::Consuming::Helpers
|
9
|
+
|
10
|
+
def initialize(connection)
|
11
|
+
self.connection = connection
|
12
|
+
self.channel = connection.create_channel
|
13
|
+
end
|
14
|
+
|
15
|
+
def retry_errors
|
16
|
+
logger.info "RabbitMQ RetryWorker: starting"
|
17
|
+
channel.open if channel.closed?
|
18
|
+
error_queue.message_count.times do
|
19
|
+
delivery_info, properties, payload = error_queue.pop(
|
20
|
+
{:manual_ack => true}
|
21
|
+
)
|
22
|
+
retry_message(delivery_info, properties, payload)
|
23
|
+
end
|
24
|
+
channel.close
|
25
|
+
logger.info "RabbitMQ RetryWorker: done"
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
attr_accessor :connection, :channel
|
31
|
+
|
32
|
+
def error_queue
|
33
|
+
channel.queue(
|
34
|
+
"emque.#{config.app_name}.error",
|
35
|
+
:durable => true,
|
36
|
+
:auto_delete => false,
|
37
|
+
:arguments => {
|
38
|
+
"x-dead-letter-exchange" => "#{config.app_name}.error"
|
39
|
+
}
|
40
|
+
)
|
41
|
+
end
|
42
|
+
|
43
|
+
def retry_message(delivery_info, metadata, payload)
|
44
|
+
begin
|
45
|
+
logger.info "RabbitMQ RetryWorker: processing message #{payload}"
|
46
|
+
message = Emque::Consuming::Message.new(
|
47
|
+
:original => payload
|
48
|
+
)
|
49
|
+
::Emque::Consuming::Consumer.new.consume(:process, message)
|
50
|
+
channel.ack(delivery_info.delivery_tag)
|
51
|
+
rescue StandardError
|
52
|
+
channel.nack(delivery_info.delivery_tag)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require "bunny"
|
2
|
+
|
3
|
+
module Emque
|
4
|
+
module Consuming
|
5
|
+
module Adapters
|
6
|
+
module RabbitMq
|
7
|
+
class Worker
|
8
|
+
include Emque::Consuming::Actor
|
9
|
+
trap_exit :actor_died
|
10
|
+
|
11
|
+
attr_accessor :topic
|
12
|
+
|
13
|
+
def actor_died(actor, reason)
|
14
|
+
unless shutdown
|
15
|
+
logger.error "#{log_prefix} actor_died - died: #{reason}"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(connection, topic)
|
20
|
+
self.topic = topic
|
21
|
+
self.name = "#{self.topic} worker"
|
22
|
+
self.shutdown = false
|
23
|
+
|
24
|
+
# @note: channels are not thread safe, so is better to use
|
25
|
+
# a new channel in each worker.
|
26
|
+
# https://github.com/jhbabon/amqp-celluloid/blob/master/lib/consumer.rb
|
27
|
+
self.channel = connection.create_channel
|
28
|
+
|
29
|
+
if config.adapter.options[:prefetch]
|
30
|
+
channel.prefetch(config.adapter.options[:prefetch])
|
31
|
+
end
|
32
|
+
|
33
|
+
self.queue =
|
34
|
+
channel
|
35
|
+
.queue(
|
36
|
+
"emque.#{config.app_name}.#{topic}",
|
37
|
+
:durable => config.adapter.options[:durable],
|
38
|
+
:auto_delete => config.adapter.options[:auto_delete],
|
39
|
+
:arguments => {
|
40
|
+
"x-dead-letter-exchange" => "#{config.app_name}.error"
|
41
|
+
}
|
42
|
+
)
|
43
|
+
.bind(
|
44
|
+
channel.fanout(topic, :durable => true, :auto_delete => false)
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
def start
|
49
|
+
logger.info "#{log_prefix} starting..."
|
50
|
+
queue.subscribe(:manual_ack => true, &method(:process_message))
|
51
|
+
logger.debug "#{log_prefix} started"
|
52
|
+
end
|
53
|
+
|
54
|
+
def stop
|
55
|
+
logger.debug "#{log_prefix} stopping..."
|
56
|
+
super do
|
57
|
+
logger.debug "#{log_prefix} closing channel"
|
58
|
+
channel.close
|
59
|
+
end
|
60
|
+
logger.debug "#{log_prefix} stopped"
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
attr_accessor :name, :channel, :queue
|
66
|
+
|
67
|
+
def process_message(delivery_info, metadata, payload)
|
68
|
+
begin
|
69
|
+
logger.info "#{log_prefix} processing message #{payload}"
|
70
|
+
message = Emque::Consuming::Message.new(
|
71
|
+
:original => payload
|
72
|
+
)
|
73
|
+
::Emque::Consuming::Consumer.new.consume(:process, message)
|
74
|
+
channel.ack(delivery_info.delivery_tag)
|
75
|
+
rescue StandardError
|
76
|
+
channel.nack(delivery_info.delivery_tag)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def log_prefix
|
81
|
+
"RabbitMQ Worker: #{object_id} #{name}"
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|