chasqui 0.9.3 → 1.0.0.pre.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/README.md +60 -39
- data/Vagrantfile +15 -0
- data/lib/chasqui.rb +28 -55
- data/lib/chasqui/broker.rb +3 -3
- data/lib/chasqui/{multi_broker.rb → brokers/redis_broker.rb} +11 -11
- data/lib/chasqui/config.rb +26 -6
- data/lib/chasqui/queue_adapter.rb +14 -0
- data/lib/chasqui/queue_adapters/redis_queue_adapter.rb +51 -0
- data/lib/chasqui/subscriber.rb +56 -49
- data/lib/chasqui/subscriptions.rb +67 -0
- data/lib/chasqui/version.rb +1 -1
- data/lib/chasqui/worker.rb +81 -0
- data/spec/integration/pubsub_examples.rb +20 -20
- data/spec/integration/resque_spec.rb +1 -1
- data/spec/integration/setup/subscribers.rb +25 -9
- data/spec/integration/sidekiq_spec.rb +1 -1
- data/spec/lib/chasqui/{multi_broker_spec.rb → brokers/redis_broker_spec.rb} +54 -26
- data/spec/lib/chasqui/cli_spec.rb +1 -1
- data/spec/lib/chasqui/config_spec.rb +121 -0
- data/spec/lib/chasqui/fake_queue_adapter_spec.rb +5 -0
- data/spec/lib/chasqui/queue_adapters/redis_queue_adapter_spec.rb +73 -0
- data/spec/lib/chasqui/subscriber_spec.rb +95 -43
- data/spec/lib/chasqui/subscriptions_spec.rb +60 -0
- data/spec/lib/chasqui/worker_spec.rb +96 -0
- data/spec/lib/chasqui_spec.rb +32 -191
- data/spec/spec_helper.rb +1 -1
- data/spec/support/chasqui_spec_helpers.rb +28 -0
- data/spec/support/fake_queue_adapter.rb +3 -0
- data/spec/support/fake_subscriber.rb +2 -1
- data/spec/support/shared_examples/queue_adapter_examples.rb +13 -0
- metadata +25 -18
- data/lib/chasqui/subscription.rb +0 -53
- data/lib/chasqui/workers/resque_worker.rb +0 -25
- data/lib/chasqui/workers/sidekiq_worker.rb +0 -45
- data/lib/chasqui/workers/worker.rb +0 -34
- data/spec/lib/chasqui/subscription_spec.rb +0 -35
- data/spec/lib/chasqui/workers/resque_worker_spec.rb +0 -27
- data/spec/lib/chasqui/workers/sidekiq_worker_spec.rb +0 -34
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2a3b1a24a648748e026c9938d71f77aeda97a7f
|
4
|
+
data.tar.gz: cf1b2e7b5d261c31980b46edc42fcff4048374a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bedd779b43a73f7df0021bf31c19aa686093432e9488f7b1d32be64a066defc788b040596d19a662ddca6715245c36f0d2eb5a810b3ad26eaaf2d05ceceefdf1
|
7
|
+
data.tar.gz: a06828b09503066ea658447bcc0af0099339be201538421e94923bf138c8d25926eae97264c99291aca5259be3dfcf53fccf72ca6acf8f580ab092322f1ae48f
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -3,11 +3,13 @@
|
|
3
3
|
|
4
4
|
# Chasqui
|
5
5
|
|
6
|
-
Chasqui is a simple, lightweight, persistent implementation of the
|
7
|
-
messaging pattern for service oriented
|
6
|
+
Chasqui is a simple, lightweight, persistent implementation of the
|
7
|
+
publish-subscribe (pub-sub) messaging pattern for service oriented
|
8
|
+
architectures.
|
8
9
|
|
9
|
-
Chasqui delivers messages to subscribers in a Resque-compatible format. If you
|
10
|
-
using Resque and/or Sidekiq, Chasqui will make a wonderful
|
10
|
+
Chasqui delivers messages to subscribers in a Resque-compatible format. If you
|
11
|
+
are already using Resque and/or Sidekiq, Chasqui will make a wonderful
|
12
|
+
companion to your architecture.
|
11
13
|
|
12
14
|
## Installation
|
13
15
|
|
@@ -25,22 +27,26 @@ Or install it yourself as:
|
|
25
27
|
|
26
28
|
## Dependencies
|
27
29
|
|
28
|
-
Chasqui uses Redis to queue events and manage
|
29
|
-
|
30
|
-
you prefer, you can run `vagrant up` to run
|
30
|
+
Chasqui uses [Redis](http://redis.io/) to queue events and manage
|
31
|
+
subscriptions. You can install Redis with your favorite package manager, such
|
32
|
+
as homebrew, yum, or apt, or if you prefer, you can run `vagrant up` to run
|
33
|
+
Redis in a virtual machine.
|
31
34
|
|
32
35
|
## Quick Start
|
33
36
|
|
34
|
-
Chasqui
|
35
|
-
|
36
|
-
|
37
|
+
Chasqui consists of two components - a client and a broker. Clients can both
|
38
|
+
publish to a channel and subscribe to events on one or more channels. The
|
39
|
+
broker transforms incoming events into Resque (or Sidekiq) jobs and places them
|
40
|
+
on one or more queues according to the currently registered subscribers. Under
|
41
|
+
the hood, a subscriber is simply a Sidekiq/Resque worker that processes jobs
|
42
|
+
placed on a queue.
|
37
43
|
|
38
44
|
### Start the broker
|
39
45
|
|
40
46
|
chasqui -r redis://localhost:6379/0 -q my-app
|
41
47
|
|
42
|
-
Your broker must use the same
|
43
|
-
|
48
|
+
Your broker must use the same Redis connection as your Sidekiq/Resque workers.
|
49
|
+
For a list of available broker options, see `chasqui --help`.
|
44
50
|
|
45
51
|
### Publish events
|
46
52
|
|
@@ -48,8 +54,7 @@ Publishing events is simple.
|
|
48
54
|
|
49
55
|
# file: publisher.rb
|
50
56
|
require 'chasqui'
|
51
|
-
Chasqui.publish 'user.sign-up', '
|
52
|
-
Chasqui.publish 'user.cancel', 'Dart Vader', 'invalid use of the force'
|
57
|
+
Chasqui.publish 'user.sign-up', name: 'Darth Vader'
|
53
58
|
|
54
59
|
Be sure to run the publisher, broker, and subscribers in separate terminal
|
55
60
|
windows.
|
@@ -58,34 +63,35 @@ windows.
|
|
58
63
|
|
59
64
|
### Subscribe to events
|
60
65
|
|
61
|
-
Subscribing to events is also simple.
|
62
|
-
|
63
|
-
generate the appropriate worker class. Within the subscriber block, you define
|
64
|
-
one or more `on` blocks in which you place your application logic for handling
|
65
|
-
an event.
|
66
|
+
Subscribing to events is also simple. In the following example, we create a
|
67
|
+
subscriber to handle events published to the `user.sign-up` channel.
|
66
68
|
|
67
69
|
# file: subscriber1.rb
|
68
70
|
require 'chasqui'
|
69
71
|
|
70
|
-
|
72
|
+
class UserSignUpSubscriber
|
73
|
+
include Chasqui::Subscriber
|
74
|
+
subscribe channel: 'user.sign-up'
|
71
75
|
|
72
|
-
|
73
|
-
#
|
76
|
+
def perform(payload)
|
77
|
+
# Do something when the user signs up.
|
78
|
+
#
|
79
|
+
# User.create(name: payload[:user])
|
80
|
+
# => #<User:0X00fe346 @name="Darth Vader">
|
74
81
|
end
|
75
|
-
|
76
|
-
on 'user.cancel' do |user_id, reason|
|
77
|
-
# do something else when user cancels
|
78
|
-
end
|
79
|
-
|
80
82
|
end
|
81
83
|
|
82
|
-
|
83
|
-
its own unique queue name__.
|
84
|
+
### Running Sidekiq subscribers
|
84
85
|
|
85
86
|
Here is how you can run the subscriber as a sidekiq worker:
|
86
87
|
|
87
88
|
sidekiq -r subscriber.rb
|
88
89
|
|
90
|
+
For more information on running Sidekiq, please refer to the
|
91
|
+
[Sidekiq documentation](https://github.com/mperham/sidekiq).
|
92
|
+
|
93
|
+
### Running Resque subscribers
|
94
|
+
|
89
95
|
To run the resque worker, you first need to create a Rakefile.
|
90
96
|
|
91
97
|
# Rakefile
|
@@ -104,25 +110,40 @@ Then you can run the resque worker to start processing events.
|
|
104
110
|
|
105
111
|
rake resque:work
|
106
112
|
|
113
|
+
For more information on running Resque workers, please refer to
|
114
|
+
the [resque documentation](https://github.com/resque/resque).
|
115
|
+
|
107
116
|
## Why Chasqui?
|
108
117
|
|
109
|
-
* Reduces coupling between applications
|
110
|
-
*
|
111
|
-
*
|
118
|
+
* Reduces coupling between applications.
|
119
|
+
* Simple to learn and use.
|
120
|
+
* Integrates with the popular Sidekiq and Resque background worker libraries.
|
121
|
+
* Queues events for registered subscribers even if a subscriber is unavailable.
|
122
|
+
* Subscribers can benefit from Sidekiq's built-in retry functionality for
|
123
|
+
failed jobs.
|
112
124
|
|
113
125
|
## Limitations
|
114
126
|
|
115
|
-
In order for
|
116
|
-
subscribers must connect to the same Redis database.
|
127
|
+
In order for Chasqui to work properly, the publisher, broker, and all
|
128
|
+
subscribers must connect to the same Redis database. Chasqui is intentionally
|
129
|
+
simple and may not have all of the features you would expect from a messaging
|
130
|
+
system. If Chasqui is missing a feature that you think it should have, please
|
131
|
+
consider [opening a GitHub issue](https://github.com/jbgo/chasqui/issues/new)
|
132
|
+
to discuss your feature proposal.
|
117
133
|
|
118
134
|
## Contributing
|
119
135
|
|
120
|
-
* For new functionality, please open an issue for discussion before creating a
|
121
|
-
|
136
|
+
* For new functionality, please open an issue for discussion before creating a
|
137
|
+
pull request.
|
138
|
+
* For bug fixes, you are welcome to create a pull request without first opening
|
139
|
+
an issue.
|
122
140
|
* Except for documentation changes, tests are required with all pull requests.
|
123
|
-
* Please be polite and respectful when discussing the project with maintainers
|
141
|
+
* Please be polite and respectful when discussing the project with maintainers
|
142
|
+
and your fellow contributors.
|
124
143
|
|
125
144
|
## Code of Conduct
|
126
145
|
|
127
|
-
If you are unsure whether or not your communication may be inappropriate,
|
128
|
-
|
146
|
+
If you are unsure whether or not your communication may be inappropriate,
|
147
|
+
please consult the [Chasqui Code of Conduct](code-of-conduct.md). If you even
|
148
|
+
suspect harassment or abuse, please report it to the email address listed in
|
149
|
+
the Code of Conduct.
|
data/Vagrantfile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# -*- mode: ruby -*-
|
2
|
+
# vi: set ft=ruby :
|
3
|
+
|
4
|
+
Vagrant.configure(2) do |config|
|
5
|
+
config.vm.box = 'hashicorp/precise64'
|
6
|
+
|
7
|
+
config.vm.network 'forwarded_port', guest: 6379, host: 6379
|
8
|
+
|
9
|
+
config.vm.provision "shell", inline: <<-SHELL
|
10
|
+
sudo apt-get update
|
11
|
+
sudo apt-get install -y redis-server
|
12
|
+
sudo sed -ie 's/bind 127.0.0.1/bind 0.0.0.0/' /etc/redis/redis.conf
|
13
|
+
sudo service redis-server restart
|
14
|
+
SHELL
|
15
|
+
end
|
data/lib/chasqui.rb
CHANGED
@@ -7,88 +7,61 @@ require 'redis-namespace'
|
|
7
7
|
require "chasqui/version"
|
8
8
|
require "chasqui/config"
|
9
9
|
require "chasqui/broker"
|
10
|
-
require "chasqui/
|
10
|
+
require "chasqui/brokers/redis_broker"
|
11
|
+
require "chasqui/queue_adapter"
|
12
|
+
require "chasqui/queue_adapters/redis_queue_adapter"
|
11
13
|
require "chasqui/subscriber"
|
12
|
-
require "chasqui/
|
13
|
-
require "chasqui/
|
14
|
-
require "chasqui/workers/resque_worker"
|
15
|
-
require "chasqui/workers/sidekiq_worker"
|
14
|
+
require "chasqui/subscriptions"
|
15
|
+
require "chasqui/worker"
|
16
16
|
|
17
17
|
module Chasqui
|
18
|
+
|
19
|
+
class ConfigurationError < StandardError; end
|
20
|
+
|
18
21
|
class << self
|
19
22
|
extend Forwardable
|
20
|
-
def_delegators :config,
|
23
|
+
def_delegators :config, *CONFIG_SETTINGS
|
24
|
+
def_delegators :subscriptions, :register, :unregister
|
21
25
|
|
22
26
|
def configure(&block)
|
23
|
-
|
24
|
-
yield @config
|
27
|
+
yield config
|
25
28
|
end
|
26
29
|
|
27
30
|
def config
|
28
31
|
@config ||= Config.new
|
29
32
|
end
|
30
33
|
|
31
|
-
def publish(
|
32
|
-
redis.lpush inbox_queue,
|
33
|
-
end
|
34
|
-
|
35
|
-
def subscribe(options={}, &block)
|
36
|
-
queue = options.fetch :queue
|
37
|
-
channel = options.fetch :channel, config.channel
|
38
|
-
|
39
|
-
create_subscription(queue, channel).tap do |subscription|
|
40
|
-
subscription.subscriber.evaluate(&block) if block_given?
|
41
|
-
redis.sadd subscription_key(channel), subscription.subscription_id
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def unsubscribe(channel, options={}, &block)
|
46
|
-
queue = options.fetch :queue
|
47
|
-
subscription = subscriptions[queue.to_s]
|
48
|
-
|
49
|
-
if subscription
|
50
|
-
redis.srem subscription_key(channel), subscription.subscription_id
|
51
|
-
subscription.subscription_id
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def subscription(queue)
|
56
|
-
subscriptions[queue.to_s]
|
34
|
+
def publish(channel, *args)
|
35
|
+
redis.lpush inbox_queue, build_event(channel, *args).to_json
|
57
36
|
end
|
58
37
|
|
59
|
-
def
|
60
|
-
|
61
|
-
"Subscriber__#{queue_name_constant}".to_sym
|
62
|
-
end
|
63
|
-
|
64
|
-
def subscription_key(channel)
|
65
|
-
"subscriptions:#{channel}"
|
38
|
+
def subscriptions
|
39
|
+
@subscriptions ||= Subscriptions.new build_queue_adapter
|
66
40
|
end
|
67
41
|
|
68
42
|
private
|
69
43
|
|
70
|
-
def
|
71
|
-
subscriptions[queue.to_s] ||= Subscription.new queue, channel
|
72
|
-
end
|
73
|
-
|
74
|
-
def subscriptions
|
75
|
-
@subscriptions ||= {}
|
76
|
-
end
|
77
|
-
|
78
|
-
def build_payload(event, *args)
|
44
|
+
def build_event(channel, *args)
|
79
45
|
opts = extract_job_options!(*args)
|
80
46
|
|
81
|
-
payload = {
|
82
|
-
payload[:retry] = opts
|
47
|
+
payload = { channel: channel, payload: args }
|
48
|
+
payload[:retry] = fetch_option(opts, :retry, true) || false
|
83
49
|
payload[:created_at] = Time.now.to_f.to_s
|
84
50
|
|
85
51
|
payload
|
86
52
|
end
|
87
53
|
|
88
54
|
def extract_job_options!(*args)
|
89
|
-
if args.last.kind_of?(Hash)
|
90
|
-
|
91
|
-
|
55
|
+
opts = args.last.delete(:job_options) if args.last.kind_of?(Hash)
|
56
|
+
opts ||= {}
|
57
|
+
end
|
58
|
+
|
59
|
+
def fetch_option(opts, key, default=nil)
|
60
|
+
opts.fetch key.to_sym, opts.fetch(key.to_s, default)
|
61
|
+
end
|
62
|
+
|
63
|
+
def build_queue_adapter
|
64
|
+
queue_adapter.new
|
92
65
|
end
|
93
66
|
end
|
94
67
|
end
|
data/lib/chasqui/broker.rb
CHANGED
@@ -4,7 +4,7 @@ class Chasqui::Broker
|
|
4
4
|
attr_reader :config, :redis, :redis_namespace
|
5
5
|
|
6
6
|
extend Forwardable
|
7
|
-
def_delegators :@config, :
|
7
|
+
def_delegators :@config, :inbox_queue, :logger
|
8
8
|
|
9
9
|
ShutdownSignals = %w(INT QUIT ABRT TERM).freeze
|
10
10
|
|
@@ -24,7 +24,7 @@ class Chasqui::Broker
|
|
24
24
|
install_signal_handlers
|
25
25
|
|
26
26
|
logger.info "broker started with pid #{Process.pid}"
|
27
|
-
logger.info "configured to fetch events from #{
|
27
|
+
logger.info "configured to fetch events from #{inbox_queue} on #{redis.inspect}"
|
28
28
|
|
29
29
|
until_shutdown_requested { forward_event }
|
30
30
|
end
|
@@ -35,7 +35,7 @@ class Chasqui::Broker
|
|
35
35
|
|
36
36
|
class << self
|
37
37
|
def start
|
38
|
-
Chasqui::
|
38
|
+
Chasqui::RedisBroker.new.start
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'securerandom'
|
2
2
|
|
3
|
-
class Chasqui::
|
3
|
+
class Chasqui::RedisBroker < Chasqui::Broker
|
4
4
|
|
5
5
|
def forward_event
|
6
6
|
event = receive or return
|
@@ -17,22 +17,22 @@ class Chasqui::MultiBroker < Chasqui::Broker
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def in_progress_queue
|
20
|
-
with_namespace
|
20
|
+
with_namespace inbox_queue, 'in_progress'
|
21
21
|
end
|
22
22
|
|
23
|
-
def
|
24
|
-
with_namespace
|
23
|
+
def namespaced_inbox_queue
|
24
|
+
with_namespace inbox_queue
|
25
25
|
end
|
26
26
|
|
27
|
-
def build_job(queue, event)
|
27
|
+
def build_job(queue, job_class, event)
|
28
28
|
{
|
29
|
-
class:
|
29
|
+
class: job_class,
|
30
30
|
args: [event],
|
31
31
|
queue: 'my-queue',
|
32
32
|
jid: SecureRandom.hex(12),
|
33
33
|
created_at: (event['created_at'] || Time.now).to_f,
|
34
34
|
enqueued_at: Time.now.to_f,
|
35
|
-
retry:
|
35
|
+
retry: event['retry']
|
36
36
|
}.to_json
|
37
37
|
end
|
38
38
|
|
@@ -57,7 +57,7 @@ class Chasqui::MultiBroker < Chasqui::Broker
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def dequeue
|
60
|
-
redis.brpoplpush(
|
60
|
+
redis.brpoplpush(namespaced_inbox_queue, in_progress_queue, timeout: config.broker_poll_interval).tap do |event|
|
61
61
|
if event.nil?
|
62
62
|
logger.debug "reached timeout for broker poll interval: #{config.broker_poll_interval} seconds"
|
63
63
|
end
|
@@ -65,8 +65,8 @@ class Chasqui::MultiBroker < Chasqui::Broker
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def dispatch(event, subscription_id)
|
68
|
-
backend, queue = subscription_id.split('/',
|
69
|
-
job = build_job queue, event
|
68
|
+
backend, job_class, queue = subscription_id.split('/', 3)
|
69
|
+
job = build_job queue, job_class, event
|
70
70
|
|
71
71
|
logger.debug "dispatching event queue=#{queue} backend=#{backend} job=#{job}"
|
72
72
|
|
@@ -79,7 +79,7 @@ class Chasqui::MultiBroker < Chasqui::Broker
|
|
79
79
|
end
|
80
80
|
|
81
81
|
def subscriptions_for(event)
|
82
|
-
subscription_key =
|
82
|
+
subscription_key = "subscriptions:#{event['channel']}"
|
83
83
|
redis.smembers with_namespace(subscription_key)
|
84
84
|
end
|
85
85
|
|
data/lib/chasqui/config.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
module Chasqui
|
2
2
|
|
3
3
|
Defaults = {
|
4
|
+
default_queue: 'chasqui-subscribers',
|
4
5
|
inbox_queue: 'inbox',
|
5
6
|
redis_namespace: 'chasqui',
|
6
|
-
|
7
|
-
|
7
|
+
broker_poll_interval: 3,
|
8
|
+
queue_adapter: -> { QueueAdapters::RedisQueueAdapter }
|
8
9
|
}.freeze
|
9
10
|
|
10
11
|
class ConfigurationError < StandardError
|
@@ -12,22 +13,31 @@ module Chasqui
|
|
12
13
|
|
13
14
|
CONFIG_SETTINGS = [
|
14
15
|
:broker_poll_interval,
|
15
|
-
:
|
16
|
+
:channel_prefix,
|
17
|
+
:default_queue,
|
16
18
|
:inbox_queue,
|
17
19
|
:logger,
|
20
|
+
:queue_adapter,
|
18
21
|
:redis,
|
19
22
|
:worker_backend
|
20
23
|
]
|
21
24
|
|
22
25
|
class Config < Struct.new(*CONFIG_SETTINGS)
|
23
|
-
def
|
24
|
-
self[:
|
26
|
+
def default_queue
|
27
|
+
self[:default_queue] ||= Defaults.fetch(:default_queue)
|
25
28
|
end
|
26
29
|
|
27
30
|
def inbox_queue
|
28
31
|
self[:inbox_queue] ||= Defaults.fetch(:inbox_queue)
|
29
32
|
end
|
30
|
-
|
33
|
+
|
34
|
+
def queue_adapter
|
35
|
+
self[:queue_adapter] ||= Defaults.fetch(:queue_adapter).call
|
36
|
+
end
|
37
|
+
|
38
|
+
def worker_backend
|
39
|
+
self[:worker_backend] ||= choose_worker_backend
|
40
|
+
end
|
31
41
|
|
32
42
|
def redis
|
33
43
|
unless self[:redis]
|
@@ -74,5 +84,15 @@ module Chasqui
|
|
74
84
|
def broker_poll_interval
|
75
85
|
self[:broker_poll_interval] ||= Defaults.fetch(:broker_poll_interval)
|
76
86
|
end
|
87
|
+
|
88
|
+
private
|
89
|
+
|
90
|
+
def choose_worker_backend
|
91
|
+
if Object.const_defined? :Sidekiq
|
92
|
+
:sidekiq
|
93
|
+
elsif Object.const_defined? :Resque
|
94
|
+
:resque
|
95
|
+
end
|
96
|
+
end
|
77
97
|
end
|
78
98
|
end
|