proletariat 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/Gemfile +8 -0
- data/README.md +93 -0
- data/Rakefile +1 -0
- data/lib/proletariat/concerns/logging.rb +17 -0
- data/lib/proletariat/cucumber.rb +20 -0
- data/lib/proletariat/manager.rb +99 -0
- data/lib/proletariat/publisher.rb +87 -0
- data/lib/proletariat/queue_config.rb +26 -0
- data/lib/proletariat/runner.rb +149 -0
- data/lib/proletariat/subscriber.rb +249 -0
- data/lib/proletariat/testing/expectation.rb +15 -0
- data/lib/proletariat/testing/expectation_guarantor.rb +145 -0
- data/lib/proletariat/testing/fixnum_extension.rb +10 -0
- data/lib/proletariat/testing.rb +41 -0
- data/lib/proletariat/version.rb +4 -0
- data/lib/proletariat/worker.rb +131 -0
- data/lib/proletariat.rb +87 -0
- data/proletariat.gemspec +24 -0
- data/spec/lib/proletariat_spec.rb +57 -0
- metadata +121 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: ccf124f3fb011f0dd8f06666daa0f8748c70fe32
|
4
|
+
data.tar.gz: a601e7e559d77d40b30f91be6824d4d51b13af3b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 22cd707dcdcbfbefbff38a8c4c4bf74cceefa87c42c30706f9028a89d2039fc0095ac1deff157f5efa9c4e60c881e4fe6cd2b8f10596142ef257cfbb54460cdb
|
7
|
+
data.tar.gz: 0160aced5ba612299ddb023b8cf19c361a2d02e9212bfb8c2e1d79dfbe2db07d017378ba7b9b029d894e7d18903ee371ec34623cef2941e611e4f80148a34fe6
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
# Proletariat: Background Workers Unite!
|
2
|
+
|
3
|
+
Lightweight background processing in Ruby powered by RabbitMQ and the excellent concurrent-ruby gem.
|
4
|
+
|
5
|
+
### Warning!
|
6
|
+
|
7
|
+
This software is early-alpha, may contain bugs and change considerably in the near future.
|
8
|
+
|
9
|
+
For production use I recommend the better supported and more fully-featured [Sneakers gem](https://github.com/jondot/sneakers).
|
10
|
+
|
11
|
+
## Installation
|
12
|
+
|
13
|
+
Add this line to your application's `Gemfile`:
|
14
|
+
|
15
|
+
gem 'proletariat'
|
16
|
+
|
17
|
+
And run:
|
18
|
+
|
19
|
+
$ bundle
|
20
|
+
|
21
|
+
## How to use
|
22
|
+
|
23
|
+
### RabbitMQ connection config
|
24
|
+
|
25
|
+
If you aren't using default RabbitMQ connection settings, ensure the `RABBITMQ_URL` env variable is present. Here's how that might look in your `.env` if you use Foreman:
|
26
|
+
|
27
|
+
RABBITMQ_URL=amqp://someuser:somepass@127.0.0.1/another_vhost
|
28
|
+
|
29
|
+
### Setting up a Worker
|
30
|
+
|
31
|
+
Your worker classes should inherit from `Proletariat::Worker` and implement the `#work` method.
|
32
|
+
|
33
|
+
Proletariat works exclusively on RabbitMQ Topic exchanges and routing keys can be bound via a call to `.listen_on`. This can be called multiple times to bind to multiple keys.
|
34
|
+
|
35
|
+
The `#work` method should return `:ok` on success or `:drop` / `:requeue` on failure.
|
36
|
+
|
37
|
+
Here's a complete example:
|
38
|
+
|
39
|
+
class SendUserIntroductoryEmail < Proletariat::Worker
|
40
|
+
listen_on 'user.created'
|
41
|
+
|
42
|
+
def work(message)
|
43
|
+
params = JSON.parse(message)
|
44
|
+
|
45
|
+
UserMailer.introductory_email(params).deliver!
|
46
|
+
|
47
|
+
publish 'email_sent.user.introductory', {id: params['id']}.to_json
|
48
|
+
|
49
|
+
:ok
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
### Select your Workers
|
54
|
+
|
55
|
+
If you are using Rails just create a `proletariat.rb` file in your initializers directory.
|
56
|
+
|
57
|
+
Proletariat.configure worker_classes: [SendUserIntroductoryEmail, SomeOtherWorker]
|
58
|
+
Proletariat.run!
|
59
|
+
|
60
|
+
Or define the `WORKERS` env variable.
|
61
|
+
|
62
|
+
WORKERS=SendUserIntroductoryEmail,SomeOtherWorker
|
63
|
+
|
64
|
+
### Testing with Cucumber
|
65
|
+
|
66
|
+
Add the following to your `env.rb`:
|
67
|
+
|
68
|
+
require 'proletariat/cucumber'
|
69
|
+
|
70
|
+
Use the provided helpers in your step definitions to synchronize your test suite with your workers without sacrificing the ability to test in production-like environment:
|
71
|
+
|
72
|
+
When(/^I submit a valid 'register user' form$/) do
|
73
|
+
wait_for message.on_topic('email_sent.user.introductory') do
|
74
|
+
visit ...
|
75
|
+
fill_in ...
|
76
|
+
submit ...
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
Then(/^the user should receive an introductory email$/) do
|
81
|
+
expect(unread_emails_for(new_user_email).size).to eq 1
|
82
|
+
end
|
83
|
+
|
84
|
+
## FAQ
|
85
|
+
|
86
|
+
#### Why build another RabbitMQ background worker library?
|
87
|
+
|
88
|
+
I wanted a library which shared one RabbitMQ connection across all of the workers on a given process. Many hosted RabbitMQ platforms tightly limit the max number of connections.
|
89
|
+
|
90
|
+
## TODO
|
91
|
+
- Improve test suite :(
|
92
|
+
- Add command line interface
|
93
|
+
- Abstract retry strategies
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Proletariat
|
2
|
+
module Concerns
|
3
|
+
# Public: Mixin to handle logging concerns.
|
4
|
+
module Logging
|
5
|
+
# Public: Logs info to the process-wide logger.
|
6
|
+
#
|
7
|
+
# message - The message to be logged.
|
8
|
+
#
|
9
|
+
# Returns nil.
|
10
|
+
def log_info(message)
|
11
|
+
Proletariat.logger.info "#{self.class.name} #{object_id}: #{message}"
|
12
|
+
|
13
|
+
nil
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'proletariat/testing'
|
2
|
+
|
3
|
+
# Mix testing helpers into World.
|
4
|
+
World(Proletariat::Testing)
|
5
|
+
|
6
|
+
# Hide logs by default.
|
7
|
+
AfterConfiguration do |config|
|
8
|
+
Proletariat.logger = Logger.new('/dev/null')
|
9
|
+
end
|
10
|
+
|
11
|
+
# Ensure Proletariat running before each test.
|
12
|
+
Before do
|
13
|
+
Proletariat.run! unless Proletariat.running?
|
14
|
+
end
|
15
|
+
|
16
|
+
# Stop workers and purge queues between scenarios.
|
17
|
+
After do
|
18
|
+
Proletariat.stop
|
19
|
+
Proletariat.purge
|
20
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
module Proletariat
|
2
|
+
# Public: Maintains a pool of worker threads and a RabbitMQ subscriber
|
3
|
+
# thread. Uses information from the worker class to generate queue
|
4
|
+
# config.
|
5
|
+
class Manager < Concurrent::Supervisor
|
6
|
+
# Public: Creates a new Manager instance.
|
7
|
+
#
|
8
|
+
# connection - An open Bunny::Session object.
|
9
|
+
# exchange_name - A String of the RabbitMQ topic exchange.
|
10
|
+
# worker_class - A subclass of Proletariat::Worker to handle messages.
|
11
|
+
# options - A Hash of additional optional parameters (default: {}):
|
12
|
+
# :worker_threads - The size of the worker thread pool.
|
13
|
+
def initialize(connection, exchange_name, worker_class, options = {})
|
14
|
+
super()
|
15
|
+
|
16
|
+
@connection = connection
|
17
|
+
@exchange_name = exchange_name
|
18
|
+
@worker_class = worker_class
|
19
|
+
@worker_threads = options.fetch :worker_threads, 3
|
20
|
+
|
21
|
+
create_worker_pool
|
22
|
+
create_subscriber
|
23
|
+
|
24
|
+
worker_pool.each { |worker| add_worker worker }
|
25
|
+
add_worker subscriber
|
26
|
+
end
|
27
|
+
|
28
|
+
# Public: Purge the RabbitMQ queue.
|
29
|
+
#
|
30
|
+
# Returns nil.
|
31
|
+
def purge
|
32
|
+
subscriber.purge
|
33
|
+
|
34
|
+
nil
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
# Internal: Returns an open Bunny::Session object.
|
40
|
+
attr_reader :connection
|
41
|
+
|
42
|
+
# Internal: Returns the name of the RabbitMQ topic exchange.
|
43
|
+
attr_reader :exchange_name
|
44
|
+
|
45
|
+
# Internal: Returns the Subscriber actor for this Manager.
|
46
|
+
attr_reader :subscriber
|
47
|
+
|
48
|
+
# Internal: Returns the subclass of Worker used to process messages.
|
49
|
+
attr_reader :worker_class
|
50
|
+
|
51
|
+
# Internal: Returns the pool of initialized workers.
|
52
|
+
attr_reader :worker_pool
|
53
|
+
|
54
|
+
# Internal: Returns a shared mailbox for the pool of workers.
|
55
|
+
attr_reader :workers_mailbox
|
56
|
+
|
57
|
+
# Internal: Returns the number of worker threads in the worker pool.
|
58
|
+
attr_reader :worker_threads
|
59
|
+
|
60
|
+
# Internal: Assign a new Subscriber instance (configured for the current
|
61
|
+
# worker type) to the manager's subscriber property.
|
62
|
+
#
|
63
|
+
# Returns nil.
|
64
|
+
def create_subscriber
|
65
|
+
@subscriber = Subscriber.new(
|
66
|
+
connection,
|
67
|
+
workers_mailbox,
|
68
|
+
generate_queue_config
|
69
|
+
)
|
70
|
+
|
71
|
+
nil
|
72
|
+
end
|
73
|
+
|
74
|
+
# Internal: Assign new Concurrent::Poolbox and Array[Worker] to the
|
75
|
+
# manager's workers_mailbox and worker_pool properties
|
76
|
+
# respectively.
|
77
|
+
#
|
78
|
+
# Returns nil.
|
79
|
+
def create_worker_pool
|
80
|
+
@workers_mailbox, @worker_pool = worker_class.pool(worker_threads)
|
81
|
+
|
82
|
+
nil
|
83
|
+
end
|
84
|
+
|
85
|
+
# Internal: Builds a new QueueConfig object passing in some settings from
|
86
|
+
# worker_class.
|
87
|
+
#
|
88
|
+
# Returns a new QueueConfig instance.
|
89
|
+
def generate_queue_config
|
90
|
+
QueueConfig.new(
|
91
|
+
worker_class.name,
|
92
|
+
exchange_name,
|
93
|
+
worker_class.routing_keys,
|
94
|
+
worker_threads,
|
95
|
+
false
|
96
|
+
)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
module Proletariat
|
2
|
+
# Public: Listens for messages in it's mailbox and publishes
|
3
|
+
# these to a RabbitMQ topic exchange.
|
4
|
+
class Publisher < Concurrent::Actor
|
5
|
+
include Concerns::Logging
|
6
|
+
|
7
|
+
# Public: Creates a new Publisher instance.
|
8
|
+
#
|
9
|
+
# connection - An open Bunny::Session object.
|
10
|
+
# exchange_name - A String of the RabbitMQ topic exchange.
|
11
|
+
def initialize(connection, exchange_name)
|
12
|
+
@channel = connection.create_channel
|
13
|
+
@exchange = channel.topic(exchange_name, durable: true)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Public: Called by the Concurrent framework to handle new mailbox
|
17
|
+
# messages. Overridden in this subclass to push messages to a
|
18
|
+
# RabbitMQ topic exchange.
|
19
|
+
#
|
20
|
+
# to - The routing key for the message to as a String. In accordance
|
21
|
+
# with the RabbitMQ convention you can use the '*' character to
|
22
|
+
# replace one word and the '#' to replace many words.
|
23
|
+
# message - The message as a String.
|
24
|
+
#
|
25
|
+
# Returns nil.
|
26
|
+
def act(to, message)
|
27
|
+
publish(to, message)
|
28
|
+
|
29
|
+
nil
|
30
|
+
end
|
31
|
+
|
32
|
+
# Public: Called by the Concurrent framework on actor start. Overridden in
|
33
|
+
# this subclass to log the status of the publisher.
|
34
|
+
#
|
35
|
+
# Returns nil.
|
36
|
+
def on_run
|
37
|
+
super
|
38
|
+
log_info 'Now online'
|
39
|
+
|
40
|
+
nil
|
41
|
+
end
|
42
|
+
|
43
|
+
# Public: Called by the Concurrent framework on actor stop. Overridden in
|
44
|
+
# this subclass to log the status of the publisher.
|
45
|
+
def on_stop
|
46
|
+
log_info 'Attempting graceful shutdown.'
|
47
|
+
wait_for_publish_queue unless queue.empty?
|
48
|
+
|
49
|
+
super
|
50
|
+
|
51
|
+
log_info 'Now offline'
|
52
|
+
|
53
|
+
nil
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
# Internal: Returns the Bunny::Channel in use.
|
59
|
+
attr_reader :channel
|
60
|
+
|
61
|
+
# Internal: Returns the Bunny::Exchange in use.
|
62
|
+
attr_reader :exchange
|
63
|
+
|
64
|
+
# Internal: Handles the actual message send to the exchange.
|
65
|
+
#
|
66
|
+
# to - The routing key.
|
67
|
+
# message - The message as a String.
|
68
|
+
#
|
69
|
+
# Returns nil.
|
70
|
+
def publish(to, message)
|
71
|
+
exchange.publish message, routing_key: to, persistent: true
|
72
|
+
|
73
|
+
nil
|
74
|
+
end
|
75
|
+
|
76
|
+
# Internal: Blocks until each message has been published.
|
77
|
+
#
|
78
|
+
# Returns nil.
|
79
|
+
def wait_for_publish_queue
|
80
|
+
log_info 'Waiting for work queue to drain.'
|
81
|
+
|
82
|
+
publish(*queue.pop.message) until queue.empty?
|
83
|
+
|
84
|
+
nil
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# Internal: Value object to hold RabbitMQ settings.
|
2
|
+
class QueueConfig < Struct.new(:worker_name,
|
3
|
+
:exchange_name,
|
4
|
+
:routing_keys,
|
5
|
+
:prefetch,
|
6
|
+
:auto_delete)
|
7
|
+
# Public: Create an underscored RabbitMQ queue name from the worker_name.
|
8
|
+
#
|
9
|
+
# Examples
|
10
|
+
#
|
11
|
+
# config = QueueConfig.new('ExampleWorker', ...)
|
12
|
+
# config.queue_name
|
13
|
+
# # => 'example_worker'
|
14
|
+
#
|
15
|
+
# Returns the queue name as a String.
|
16
|
+
def queue_name
|
17
|
+
@queue_name ||= begin
|
18
|
+
worker_name
|
19
|
+
.gsub(/::/, '/')
|
20
|
+
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
21
|
+
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
22
|
+
.tr('-', '_')
|
23
|
+
.downcase
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,149 @@
|
|
1
|
+
module Proletariat
|
2
|
+
# Public: Sets up a supervisor which maintains a single Publisher and a
|
3
|
+
# per-worker Manager instance.
|
4
|
+
class Runner
|
5
|
+
extend Forwardable
|
6
|
+
|
7
|
+
# Public: Delegate lifecycle calls to the supervisor.
|
8
|
+
def_delegators :supervisor, :run, :run!, :stop, :running?
|
9
|
+
|
10
|
+
# Public: Returns an open Bunny::Session object.
|
11
|
+
attr_reader :connection
|
12
|
+
|
13
|
+
# Public: Returns the name of the RabbitMQ topic exchange.
|
14
|
+
attr_reader :exchange_name
|
15
|
+
|
16
|
+
# Public: Creates a new Runner instance. Default options should be fine for
|
17
|
+
# most scenarios.
|
18
|
+
#
|
19
|
+
# options - A Hash of options (default: {}):
|
20
|
+
# :connection - An open RabbitMQ::Session object.
|
21
|
+
# :exchange_name - The RabbitMQ topic exchange name as a
|
22
|
+
# String.
|
23
|
+
# :publisher_threads - The size of the publisher thread pool.
|
24
|
+
# :supervisor - A Supervisor instance.
|
25
|
+
# :worker_classes - An Array of Worker subclasses.
|
26
|
+
# :worker_threads - The size of the worker thread pool.
|
27
|
+
def initialize(options = {})
|
28
|
+
@connection = options.fetch :connection, create_connection
|
29
|
+
@exchange_name = options.fetch :exchange_name, DEFAULT_EXCHANGE_NAME
|
30
|
+
@publisher_threads = options.fetch :publisher_threads, 2
|
31
|
+
@supervisor = options.fetch :supervisor, create_supervisor
|
32
|
+
@worker_classes = options.fetch :worker_classes, []
|
33
|
+
@worker_threads = options.fetch :worker_threads, 3
|
34
|
+
|
35
|
+
@managers = []
|
36
|
+
|
37
|
+
create_publisher_pool
|
38
|
+
|
39
|
+
add_publishers_to_supervisor
|
40
|
+
add_workers_to_supervisor
|
41
|
+
end
|
42
|
+
|
43
|
+
# Public: Publishes a message to RabbitMQ via the publisher pool.
|
44
|
+
#
|
45
|
+
# to - The routing key for the message to as a String. In accordance
|
46
|
+
# with the RabbitMQ convention you can use the '*' character to
|
47
|
+
# replace one word and the '#' to replace many words.
|
48
|
+
# message - The message as a String.
|
49
|
+
#
|
50
|
+
# Returns nil.
|
51
|
+
def publish(to, message)
|
52
|
+
publishers_mailbox.post to, message
|
53
|
+
|
54
|
+
nil
|
55
|
+
end
|
56
|
+
|
57
|
+
# Public: Purge the RabbitMQ queues.
|
58
|
+
#
|
59
|
+
# Returns nil.
|
60
|
+
def purge
|
61
|
+
managers.each { |manager| manager.purge }
|
62
|
+
|
63
|
+
nil
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
# Internal: Returns an Array of the currently supervised Managers.
|
69
|
+
attr_reader :managers
|
70
|
+
|
71
|
+
# Internal: Returns the pool of initialized publishers.
|
72
|
+
attr_reader :publisher_pool
|
73
|
+
|
74
|
+
# Internal: Returns a shared mailbox for the pool of publishers.
|
75
|
+
attr_reader :publishers_mailbox
|
76
|
+
|
77
|
+
# Internal: Returns the number of publisher threads in the publisher pool.
|
78
|
+
attr_reader :publisher_threads
|
79
|
+
|
80
|
+
# Internal: Returns the supervisor instance.
|
81
|
+
attr_reader :supervisor
|
82
|
+
|
83
|
+
# Internal: Returns an Array of Worker subclasses.
|
84
|
+
attr_reader :worker_classes
|
85
|
+
|
86
|
+
# Internal: Returns the number of worker threads per manager.
|
87
|
+
attr_reader :worker_threads
|
88
|
+
|
89
|
+
# Internal: Adds each publisher in the publisher_pool to the supervisor.
|
90
|
+
#
|
91
|
+
# Returns nil.
|
92
|
+
def add_publishers_to_supervisor
|
93
|
+
publisher_pool.each { |publisher| supervisor.add_worker publisher }
|
94
|
+
|
95
|
+
nil
|
96
|
+
end
|
97
|
+
|
98
|
+
# Internal: Creates a Manager per worker_class and adds these to the
|
99
|
+
# supervisor.
|
100
|
+
#
|
101
|
+
# Returns nil.
|
102
|
+
def add_workers_to_supervisor
|
103
|
+
worker_classes.each do |worker_class|
|
104
|
+
manager = create_manager(worker_class)
|
105
|
+
@managers << manager
|
106
|
+
supervisor.add_worker manager
|
107
|
+
end
|
108
|
+
|
109
|
+
nil
|
110
|
+
end
|
111
|
+
|
112
|
+
# Internal: Creates a new Bunny::Session and opens it.
|
113
|
+
#
|
114
|
+
# Returns an open Bunny::Session instance.
|
115
|
+
def create_connection
|
116
|
+
new_connection = Bunny.new
|
117
|
+
new_connection.start
|
118
|
+
|
119
|
+
new_connection
|
120
|
+
end
|
121
|
+
|
122
|
+
# Internal: Assign new Concurrent::Poolbox and Array[Publisher] to the
|
123
|
+
# manager's publishers_mailbox and publisher_pool properties
|
124
|
+
# respectively.
|
125
|
+
#
|
126
|
+
# Returns nil.
|
127
|
+
def create_publisher_pool
|
128
|
+
@publishers_mailbox, @publisher_pool = Publisher.pool(publisher_threads,
|
129
|
+
connection,
|
130
|
+
exchange_name)
|
131
|
+
end
|
132
|
+
|
133
|
+
# Internal: Creates a new Concurrent::Supervisor.
|
134
|
+
#
|
135
|
+
# Returns a Concurrent::Supervisor instance.
|
136
|
+
def create_supervisor
|
137
|
+
Concurrent::Supervisor.new
|
138
|
+
end
|
139
|
+
|
140
|
+
# Internal: Creates a Manager for a given Worker subclass adding relevant
|
141
|
+
# arguments from Runner properties.
|
142
|
+
#
|
143
|
+
# Returns a Manager instance.
|
144
|
+
def create_manager(worker_class)
|
145
|
+
Manager.new(connection, exchange_name, worker_class,
|
146
|
+
worker_threads: worker_threads)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|