queue-bus 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/.rbenv-version +1 -0
- data/.rspec +1 -0
- data/.rvmrc +2 -0
- data/Gemfile +6 -0
- data/MIT-LICENSE +20 -0
- data/README.mdown +264 -0
- data/Rakefile +1 -0
- data/lib/queue-bus.rb +62 -0
- data/lib/queue_bus/adapters/base.rb +41 -0
- data/lib/queue_bus/adapters/data.rb +65 -0
- data/lib/queue_bus/application.rb +121 -0
- data/lib/queue_bus/config.rb +98 -0
- data/lib/queue_bus/dispatch.rb +61 -0
- data/lib/queue_bus/dispatchers.rb +26 -0
- data/lib/queue_bus/driver.rb +31 -0
- data/lib/queue_bus/heartbeat.rb +109 -0
- data/lib/queue_bus/local.rb +38 -0
- data/lib/queue_bus/matcher.rb +81 -0
- data/lib/queue_bus/publisher.rb +23 -0
- data/lib/queue_bus/publishing.rb +80 -0
- data/lib/queue_bus/rider.rb +28 -0
- data/lib/queue_bus/subscriber.rb +65 -0
- data/lib/queue_bus/subscription.rb +55 -0
- data/lib/queue_bus/subscription_list.rb +53 -0
- data/lib/queue_bus/task_manager.rb +52 -0
- data/lib/queue_bus/util.rb +87 -0
- data/lib/queue_bus/version.rb +3 -0
- data/lib/queue_bus/worker.rb +14 -0
- data/lib/tasks/resquebus.rake +2 -0
- data/queue-bus.gemspec +32 -0
- data/spec/adapter/publish_at_spec.rb +48 -0
- data/spec/adapter/support.rb +15 -0
- data/spec/adapter_spec.rb +14 -0
- data/spec/application_spec.rb +152 -0
- data/spec/config_spec.rb +83 -0
- data/spec/dispatch_spec.rb +76 -0
- data/spec/driver_spec.rb +100 -0
- data/spec/heartbeat_spec.rb +44 -0
- data/spec/integration_spec.rb +53 -0
- data/spec/matcher_spec.rb +143 -0
- data/spec/publish_spec.rb +95 -0
- data/spec/publisher_spec.rb +7 -0
- data/spec/rider_spec.rb +39 -0
- data/spec/spec_helper.rb +69 -0
- data/spec/subscriber_spec.rb +268 -0
- data/spec/subscription_list_spec.rb +43 -0
- data/spec/subscription_spec.rb +53 -0
- metadata +192 -0
data/.gitignore
ADDED
data/.rbenv-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.9.3-p194
|
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.rvmrc
ADDED
data/Gemfile
ADDED
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 Brian Leonard
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.mdown
ADDED
@@ -0,0 +1,264 @@
|
|
1
|
+
## Resque Bus
|
2
|
+
|
3
|
+
This gem uses Redis and Resque to allow simple asynchronous communication between apps.
|
4
|
+
|
5
|
+
### Install
|
6
|
+
|
7
|
+
To install, include the 'resque-bus' gem and add the following to your Rakefile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
require "queue_bus/tasks"
|
11
|
+
```
|
12
|
+
|
13
|
+
### Example
|
14
|
+
|
15
|
+
Application A can publish an event
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
# config
|
19
|
+
Resque.redis = "192.168.1.1:6379"
|
20
|
+
|
21
|
+
# business logic
|
22
|
+
QueueBus.publish("user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
|
23
|
+
|
24
|
+
# or do it later
|
25
|
+
QueueBus.publish_at(1.hour.from_now, "user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
|
26
|
+
```
|
27
|
+
|
28
|
+
Application B is subscribed to events
|
29
|
+
|
30
|
+
```ruby
|
31
|
+
# config
|
32
|
+
Resque.redis = "192.168.1.1:6379"
|
33
|
+
|
34
|
+
# initializer
|
35
|
+
QueueBus.dispatch("app_b") do
|
36
|
+
# processes event on app_b_default queue
|
37
|
+
# subscribe is short-hand to subscribe to your 'default' queue and this block with process events with the name "user_created"
|
38
|
+
subscribe "user_created" do |attributes|
|
39
|
+
NameCount.find_or_create_by_name(attributes["last_name"]).increment!
|
40
|
+
end
|
41
|
+
|
42
|
+
# processes event on app_b_critical queue
|
43
|
+
# critical is short-hand to subscribe to your 'critical' queue and this block with process events with the name "user_paid"
|
44
|
+
critical "user_paid" do |attributes|
|
45
|
+
CreditCard.charge!(attributes)
|
46
|
+
end
|
47
|
+
|
48
|
+
# you can pass any queue name you would like to process from as well IE: `banana "peeled" do |attributes|`
|
49
|
+
|
50
|
+
# and regexes work as well. note that with the above configuration along with this regex,
|
51
|
+
# the following as well as the corresponding block above would both be executed
|
52
|
+
subscribe /^user_/ do |attributes|
|
53
|
+
Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
|
54
|
+
end
|
55
|
+
|
56
|
+
# the above all filter on just the event_type, but you can filter on anything
|
57
|
+
# this would be _any_ event that has a user_id and the page value of homepage regardless of bus_event_type
|
58
|
+
subscribe "my_key", { "user_id" => :present, "page" => "homepage"} do
|
59
|
+
Mixpanel.homepage_action!(attributes["action"])
|
60
|
+
end
|
61
|
+
end
|
62
|
+
```
|
63
|
+
|
64
|
+
Applications can also subscribe within classes using the provided `Subscriber` module.
|
65
|
+
|
66
|
+
```ruby
|
67
|
+
class SimpleSubscriber
|
68
|
+
include QueueBus::Subscriber
|
69
|
+
subscribe :my_method
|
70
|
+
|
71
|
+
def my_method(attributes)
|
72
|
+
# heavy lifting
|
73
|
+
end
|
74
|
+
end
|
75
|
+
```
|
76
|
+
|
77
|
+
The following is equivalent to the original initializer and shows more options:
|
78
|
+
|
79
|
+
```ruby
|
80
|
+
class OtherSubscriber
|
81
|
+
include QueueBus::Subscriber
|
82
|
+
application :app_b
|
83
|
+
|
84
|
+
subscribe :user_created
|
85
|
+
subscribe_queue :app_b_critical, :user_paid
|
86
|
+
subscribe_queue :app_b_default, :user_action, :bus_event_type => /^user_/
|
87
|
+
subscribe :homepage_method, :user_id => :present, :page => "homepage"
|
88
|
+
|
89
|
+
def user_created(attributes)
|
90
|
+
NameCount.find_or_create_by_name(attributes["last_name"]).increment!
|
91
|
+
end
|
92
|
+
|
93
|
+
def user_paid(attributes)
|
94
|
+
CreditCard.charge!(attributes)
|
95
|
+
end
|
96
|
+
|
97
|
+
def user_action(attributes)
|
98
|
+
Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
|
99
|
+
end
|
100
|
+
|
101
|
+
def homepage_method
|
102
|
+
Mixpanel.homepage_action!(attributes["action"])
|
103
|
+
end
|
104
|
+
end
|
105
|
+
```
|
106
|
+
|
107
|
+
Note: This subscribes when this class is loaded, so it needs to be in your load or otherwise referenced/required during app initialization to work properly.
|
108
|
+
|
109
|
+
### Commands
|
110
|
+
|
111
|
+
Each app needs to tell Redis about its subscriptions:
|
112
|
+
|
113
|
+
$ rake resquebus:subscribe
|
114
|
+
|
115
|
+
The subscription block is run inside a Resque worker which needs to be started for each app.
|
116
|
+
|
117
|
+
$ rake resquebus:setup resque:work
|
118
|
+
|
119
|
+
The incoming queue also needs to be processed on a dedicated or all the app servers.
|
120
|
+
|
121
|
+
$ rake resquebus:driver resque:work
|
122
|
+
|
123
|
+
If you want retry to work for subscribing apps, you should run resque-scheduler
|
124
|
+
|
125
|
+
$ rake resque:scheduler
|
126
|
+
|
127
|
+
### Adapters
|
128
|
+
|
129
|
+
QueueBus now supports multiple adapters! By default QueueBus uses Resque but you can now configure your application to use Sidekiq to drive and subscribe the bus.
|
130
|
+
|
131
|
+
First be sure to configure QueueBus to use Sidekiq early in your applications' initialization cycle:
|
132
|
+
```
|
133
|
+
QueueBus.adapter = 'Sidekiq'
|
134
|
+
```
|
135
|
+
You will be responsible for setting up the queues for your Sidekiq clients however you can get the appropriate queue names with the following tasks:
|
136
|
+
For driving applications:
|
137
|
+
```
|
138
|
+
$ rake resquebus:driver:sidekiq
|
139
|
+
```
|
140
|
+
For subscribing applications:
|
141
|
+
```
|
142
|
+
$ rake resquebus:setup:sidekiq
|
143
|
+
```
|
144
|
+
These tasks will provide the queue_names and some minimal suggestions for starting the client.
|
145
|
+
|
146
|
+
Your subscribing applications will still need to also use the appropriate rake task:
|
147
|
+
```
|
148
|
+
$ rake resquebus:subscribe:sidekiq
|
149
|
+
```
|
150
|
+
|
151
|
+
At the moment you are expected to include the Sidekiq gem in your own applications.
|
152
|
+
|
153
|
+
And yes we are planning on renaming and restructuring the project! Please contact the maintainer if you would like to add a different adapter.
|
154
|
+
|
155
|
+
### Heartbeat
|
156
|
+
|
157
|
+
We've found it useful to have the bus act like `cron`, triggering timed jobs throughout the system. Resque Bus calls this a heartbeat.
|
158
|
+
It uses resque-scheduler to trigger the events. You can enable it in your Rakefile.
|
159
|
+
|
160
|
+
```ruby
|
161
|
+
# resque.rake
|
162
|
+
namespace :resque do
|
163
|
+
task :setup => [:environment] do
|
164
|
+
QueueBus.heartbeat!
|
165
|
+
end
|
166
|
+
end
|
167
|
+
```
|
168
|
+
|
169
|
+
Or add it to your `schedule.yml` directly
|
170
|
+
|
171
|
+
```yaml
|
172
|
+
resquebus_heartbeat:
|
173
|
+
cron: "* * * * *"
|
174
|
+
class: "::QueueBus::Heartbeat"
|
175
|
+
queue: bus_incoming
|
176
|
+
description: "I publish a heartbeat_minutes event every minute"
|
177
|
+
```
|
178
|
+
|
179
|
+
It is the equivalent of doing this every minute
|
180
|
+
|
181
|
+
```ruby
|
182
|
+
seconds = minutes * (60)
|
183
|
+
hours = minutes / (60)
|
184
|
+
days = minutes / (60*24)
|
185
|
+
|
186
|
+
now = Time.at(seconds)
|
187
|
+
|
188
|
+
attributes = {}
|
189
|
+
|
190
|
+
now = Time.now
|
191
|
+
seconds = now.to_i
|
192
|
+
QueueBus.publish("hearbeat_minutes", {
|
193
|
+
"epoch_seconds" => seconds,
|
194
|
+
"epoch_minutes" => seconds / 1.minute,
|
195
|
+
"epoch_hours" => seconds / 1.hour,
|
196
|
+
"epoch_days" => seconds / 1.day,
|
197
|
+
"minute" => now.min
|
198
|
+
"hour" => now.hour
|
199
|
+
"day" => now.day
|
200
|
+
"month" => now.month
|
201
|
+
"year" => now.year
|
202
|
+
"yday" => now.yday
|
203
|
+
"wday" => now.wday
|
204
|
+
})
|
205
|
+
```
|
206
|
+
|
207
|
+
This allows you do something like this:
|
208
|
+
|
209
|
+
```ruby
|
210
|
+
QueueBus.dispatch("app_c") do
|
211
|
+
# runs at 10:20, 11:20, etc
|
212
|
+
subscribe "once_an_hour", 'bus_event_type' => 'heartbeat_minutes', 'minute' => 20 do |attributes|
|
213
|
+
Sitemap.generate!
|
214
|
+
end
|
215
|
+
|
216
|
+
# runs every five minutes
|
217
|
+
subscribe "every_five_minutes", 'bus_event_type' => 'heartbeat_minutes' do |attributes|
|
218
|
+
next unless attributes["epoch_minutes"] % 5 == 0
|
219
|
+
HealthCheck.run!
|
220
|
+
end
|
221
|
+
|
222
|
+
# runs at 8am on the first of every month
|
223
|
+
subscribe "new_month_morning", 'bus_event_type' => 'heartbeat_minutes', 'day' => 1, hour' => 8, 'minute' => 0, do |attributes|
|
224
|
+
next unless attributes["epoch_minutes"] % 5 == 0
|
225
|
+
Token.old.expire!
|
226
|
+
end
|
227
|
+
end
|
228
|
+
```
|
229
|
+
|
230
|
+
### Local Mode
|
231
|
+
|
232
|
+
For development, a local mode is provided and is specified in the configuration.
|
233
|
+
|
234
|
+
```ruby
|
235
|
+
# config
|
236
|
+
QueueBus.local_mode = :standalone
|
237
|
+
or
|
238
|
+
QueueBus.local_mode = :inline
|
239
|
+
```
|
240
|
+
|
241
|
+
Standalone mode does not require a separate resquebus:driver task to be running to process the
|
242
|
+
incoming queue. Simply publishing to the bus will distribute the incoming events
|
243
|
+
to the appropriate application specific queue. A separate resquebus:work task does
|
244
|
+
still need to be run to process these events
|
245
|
+
|
246
|
+
Inline mode skips queue processing entirely and directly dispatches the
|
247
|
+
event to the appropriate code block.
|
248
|
+
|
249
|
+
You can also say `QueueBus.local_mode = :suppress` to turn off publishing altogether.
|
250
|
+
This can be helpful inside some sort of migration, for example.
|
251
|
+
|
252
|
+
### TODO
|
253
|
+
|
254
|
+
* Sidekiq adapter
|
255
|
+
* Refactor rake tasks for resque/sidekiq
|
256
|
+
* Refactor to a storage adapter for Redis, so we can store subscription info in MySQL or something else
|
257
|
+
* Replace local modes with adapters
|
258
|
+
* There are a few spots in the code with TODO notes
|
259
|
+
* Make this not freak out in development without Redis or when Redis is down
|
260
|
+
* We might not actually need to publish in tests
|
261
|
+
* Add some rspec helpers for the apps to use: should_ post an event_publish or something along those lines
|
262
|
+
* Allow calling resquebus:setup and resquebus:driver together (append to ENV['QUEUES'], don't replace it)
|
263
|
+
|
264
|
+
Copyright (c) 2011 Brian Leonard, released under the MIT license
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
$LOAD_PATH.unshift 'lib'
|
data/lib/queue-bus.rb
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
require "queue_bus/version"
|
2
|
+
|
3
|
+
module QueueBus
|
4
|
+
|
5
|
+
autoload :Application, 'queue_bus/application'
|
6
|
+
autoload :Config, 'queue_bus/config'
|
7
|
+
autoload :Dispatch, 'queue_bus/dispatch'
|
8
|
+
autoload :Dispatchers, 'queue_bus/dispatchers'
|
9
|
+
autoload :Driver, 'queue_bus/driver'
|
10
|
+
autoload :Heartbeat, 'queue_bus/heartbeat'
|
11
|
+
autoload :Local, 'queue_bus/local'
|
12
|
+
autoload :Matcher, 'queue_bus/matcher'
|
13
|
+
autoload :Publishing, 'queue_bus/publishing'
|
14
|
+
autoload :Publisher, 'queue_bus/publisher'
|
15
|
+
autoload :Rider, 'queue_bus/rider'
|
16
|
+
autoload :Subscriber, 'queue_bus/subscriber'
|
17
|
+
autoload :Subscription, 'queue_bus/subscription'
|
18
|
+
autoload :SubscriptionList, 'queue_bus/subscription_list'
|
19
|
+
autoload :TaskManager, 'queue_bus/task_manager'
|
20
|
+
autoload :Util, 'queue_bus/util'
|
21
|
+
autoload :Worker, 'queue_bus/worker'
|
22
|
+
|
23
|
+
module Adapters
|
24
|
+
autoload :Base, 'queue_bus/adapters/base'
|
25
|
+
autoload :Data, 'queue_bus/adapters/data'
|
26
|
+
end
|
27
|
+
|
28
|
+
class << self
|
29
|
+
|
30
|
+
include Publishing
|
31
|
+
extend Forwardable
|
32
|
+
|
33
|
+
def_delegators :config, :default_app_key=, :default_app_key,
|
34
|
+
:default_queue=, :default_queue,
|
35
|
+
:local_mode=, :local_mode,
|
36
|
+
:before_publish=, :before_publish_callback,
|
37
|
+
:logger=, :logger, :log_application, :log_worker,
|
38
|
+
:hostname=, :hostname,
|
39
|
+
:adapter=, :adapter,
|
40
|
+
:incoming_queue=, :incoming_queue,
|
41
|
+
:redis
|
42
|
+
|
43
|
+
def_delegators :_dispatchers, :dispatch, :dispatchers, :dispatcher_by_key, :dispatcher_execute
|
44
|
+
|
45
|
+
protected
|
46
|
+
|
47
|
+
def reset
|
48
|
+
# used by tests
|
49
|
+
@config = nil
|
50
|
+
@_dispatchers = nil
|
51
|
+
end
|
52
|
+
|
53
|
+
def config
|
54
|
+
@config ||= ::QueueBus::Config.new
|
55
|
+
end
|
56
|
+
|
57
|
+
def _dispatchers
|
58
|
+
@_dispatchers ||= ::QueueBus::Dispatchers.new
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module QueueBus
|
2
|
+
module Adapters
|
3
|
+
class Base
|
4
|
+
# adapters need to define the NonImplemented methods in this class
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
enabled!
|
8
|
+
end
|
9
|
+
|
10
|
+
def enabled!
|
11
|
+
# called the first time we know we are using this adapter
|
12
|
+
# it would be a good spot to require the libraries you're using
|
13
|
+
raise NotImplementedError
|
14
|
+
end
|
15
|
+
|
16
|
+
def redis
|
17
|
+
# for now, we're always using redis as a storage mechanism so give us one
|
18
|
+
raise NotImplementedError
|
19
|
+
end
|
20
|
+
|
21
|
+
def enqueue(queue_name, klass, hash)
|
22
|
+
# enqueue the given class (Driver/Rider) in your queue
|
23
|
+
raise NotImplementedError
|
24
|
+
end
|
25
|
+
|
26
|
+
def enqueue_at(epoch_seconds, queue_name, klass, hash)
|
27
|
+
# enqueue the given class (Publisher) in your queue to run at given time
|
28
|
+
raise NotImplementedError
|
29
|
+
end
|
30
|
+
|
31
|
+
def setup_heartbeat!
|
32
|
+
# if possible, tell a recurring job system to publish every minute
|
33
|
+
raise NotImplementedError
|
34
|
+
end
|
35
|
+
|
36
|
+
def worker_included(base)
|
37
|
+
# optional method for including more modules in classes that work in the queue
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# a base adapter just for publishing and redis connection
|
2
|
+
module QueueBus
|
3
|
+
module Adapters
|
4
|
+
class Data < QueueBus::Adapters::Base
|
5
|
+
def enabled!
|
6
|
+
# nothing to do
|
7
|
+
end
|
8
|
+
|
9
|
+
def redis=(client)
|
10
|
+
@redis = client
|
11
|
+
end
|
12
|
+
|
13
|
+
def redis(&block)
|
14
|
+
raise "no redis instance set" unless @redis
|
15
|
+
block.call(@redis)
|
16
|
+
end
|
17
|
+
|
18
|
+
def enqueue(queue_name, klass, hash)
|
19
|
+
push(queue_name, :class => klass.to_s, :args => [hash])
|
20
|
+
end
|
21
|
+
|
22
|
+
def enqueue_at(epoch_seconds, queue_name, klass, hash)
|
23
|
+
item = delayed_job_to_hash_with_queue(queue_name, klass, [hash])
|
24
|
+
delayed_push(epoch_seconds, item)
|
25
|
+
end
|
26
|
+
|
27
|
+
def setup_heartbeat!(queue_name)
|
28
|
+
raise NotImplementedError
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def push(queue, item)
|
34
|
+
watch_queue(queue)
|
35
|
+
self.redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
|
36
|
+
end
|
37
|
+
|
38
|
+
# Used internally to keep track of which queues we've created.
|
39
|
+
# Don't call this directly.
|
40
|
+
def watch_queue(queue)
|
41
|
+
self.redis { |redis| redis.sadd(:queues, queue.to_s) }
|
42
|
+
end
|
43
|
+
|
44
|
+
# Used internally to stuff the item into the schedule sorted list.
|
45
|
+
# +timestamp+ can be either in seconds or a datetime object
|
46
|
+
# Insertion if O(log(n)).
|
47
|
+
# Returns true if it's the first job to be scheduled at that time, else false
|
48
|
+
def delayed_push(timestamp, item)
|
49
|
+
self.redis do |redis|
|
50
|
+
# First add this item to the list for this timestamp
|
51
|
+
redis.rpush("delayed:#{timestamp.to_i}", ::QueueBus::Util.encode(item))
|
52
|
+
|
53
|
+
# Now, add this timestamp to the zsets. The score and the value are
|
54
|
+
# the same since we'll be querying by timestamp, and we don't have
|
55
|
+
# anything else to store.
|
56
|
+
redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def delayed_job_to_hash_with_queue(queue, klass, args)
|
61
|
+
{:class => klass.to_s, :args => args, :queue => queue}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
module QueueBus
|
2
|
+
class Application
|
3
|
+
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def all
|
7
|
+
# note the names arent the same as we started with
|
8
|
+
::QueueBus.redis { |redis| redis.smembers(app_list_key).collect{ |val| new(val) } }
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_reader :app_key, :redis_key
|
13
|
+
|
14
|
+
|
15
|
+
def initialize(app_key)
|
16
|
+
@app_key = self.class.normalize(app_key)
|
17
|
+
@redis_key = "#{self.class.app_single_key}:#{@app_key}"
|
18
|
+
# raise error if only other chars
|
19
|
+
raise "Invalid application name" if @app_key.gsub("_", "").size == 0
|
20
|
+
end
|
21
|
+
|
22
|
+
def subscribe(subscription_list, log = false)
|
23
|
+
@subscriptions = nil
|
24
|
+
|
25
|
+
if subscription_list == nil || subscription_list.size == 0
|
26
|
+
unsubscribe
|
27
|
+
return true
|
28
|
+
end
|
29
|
+
|
30
|
+
temp_key = "temp_#{redis_key}:#{rand(999999999)}"
|
31
|
+
|
32
|
+
::QueueBus.redis do |redis|
|
33
|
+
redis_hash = subscription_list.to_redis
|
34
|
+
redis_hash.each do |key, hash|
|
35
|
+
redis.hset(temp_key, key, QueueBus::Util.encode(hash))
|
36
|
+
end
|
37
|
+
|
38
|
+
# make it the real one
|
39
|
+
redis.rename(temp_key, redis_key)
|
40
|
+
redis.sadd(self.class.app_list_key, app_key)
|
41
|
+
|
42
|
+
if log
|
43
|
+
redis.hgetall(redis_key).inspect
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
true
|
48
|
+
end
|
49
|
+
|
50
|
+
def unsubscribe
|
51
|
+
# TODO: clean up known queues?
|
52
|
+
::QueueBus.redis do |redis|
|
53
|
+
redis.srem(self.class.app_list_key, app_key)
|
54
|
+
redis.del(redis_key)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def no_connect_queue_names_for(subscriptions)
|
59
|
+
out = []
|
60
|
+
subscriptions.all.each do |sub|
|
61
|
+
queue = "#{app_key}_#{sub.queue_name}"
|
62
|
+
out << queue
|
63
|
+
end
|
64
|
+
out << "#{app_key}_default"
|
65
|
+
out.uniq
|
66
|
+
end
|
67
|
+
|
68
|
+
def subscription_matches(attributes)
|
69
|
+
out = subscriptions.matches(attributes)
|
70
|
+
out.each do |sub|
|
71
|
+
sub.app_key = self.app_key
|
72
|
+
end
|
73
|
+
out
|
74
|
+
end
|
75
|
+
|
76
|
+
def event_display_tuples
|
77
|
+
out = []
|
78
|
+
subscriptions.all.each do |sub|
|
79
|
+
out << [sub.class_name, sub.queue_name, sub.matcher.filters]
|
80
|
+
end
|
81
|
+
out
|
82
|
+
end
|
83
|
+
|
84
|
+
protected
|
85
|
+
|
86
|
+
def self.normalize(val)
|
87
|
+
val.to_s.gsub(/\W/, "_").downcase
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.app_list_key
|
91
|
+
"bus_apps"
|
92
|
+
end
|
93
|
+
|
94
|
+
def self.app_single_key
|
95
|
+
"bus_app"
|
96
|
+
end
|
97
|
+
|
98
|
+
def event_queues
|
99
|
+
::QueueBus.redis { |redis| redis.hgetall(redis_key) }
|
100
|
+
end
|
101
|
+
|
102
|
+
def subscriptions
|
103
|
+
@subscriptions ||= SubscriptionList.from_redis(read_redis_hash)
|
104
|
+
end
|
105
|
+
|
106
|
+
def read_redis_hash
|
107
|
+
out = {}
|
108
|
+
::QueueBus.redis do |redis|
|
109
|
+
redis.hgetall(redis_key).each do |key, val|
|
110
|
+
begin
|
111
|
+
out[key] = ::QueueBus::Util.decode(val)
|
112
|
+
rescue ::QueueBus::Util::DecodeException
|
113
|
+
out[key] = val
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
out
|
118
|
+
end
|
119
|
+
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
module QueueBus
|
2
|
+
class Config
|
3
|
+
def adapter=val
|
4
|
+
raise "Adapter already set to #{@adapter_instance.class.name}" if @adapter_instance
|
5
|
+
if val.is_a?(Class)
|
6
|
+
@adapter_instance = name_or_klass.new
|
7
|
+
elsif val.is_a?(::QueueBus::Adapters::Base)
|
8
|
+
@adapter_instance = val
|
9
|
+
else
|
10
|
+
class_name = ::QueueBus::Util.classify(val)
|
11
|
+
@adapter_instance = ::QueueBus::Util.constantize("::QueueBus::Adapters::#{class_name}").new
|
12
|
+
end
|
13
|
+
@adapter_instance
|
14
|
+
end
|
15
|
+
|
16
|
+
def adapter
|
17
|
+
return @adapter_instance if @adapter_instance
|
18
|
+
raise "no adapter has been set"
|
19
|
+
end
|
20
|
+
|
21
|
+
def redis(&block)
|
22
|
+
# TODO: could allow setting for non-redis adapters
|
23
|
+
adapter.redis(&block)
|
24
|
+
end
|
25
|
+
|
26
|
+
def default_app_key=val
|
27
|
+
@default_app_key = Application.normalize(val)
|
28
|
+
end
|
29
|
+
|
30
|
+
def default_app_key
|
31
|
+
@default_app_key
|
32
|
+
end
|
33
|
+
|
34
|
+
def default_queue=val
|
35
|
+
@default_queue = val
|
36
|
+
end
|
37
|
+
|
38
|
+
def default_queue
|
39
|
+
@default_queue
|
40
|
+
end
|
41
|
+
|
42
|
+
def local_mode=value
|
43
|
+
@local_mode = value
|
44
|
+
end
|
45
|
+
|
46
|
+
def local_mode
|
47
|
+
@local_mode
|
48
|
+
end
|
49
|
+
|
50
|
+
def incoming_queue=val
|
51
|
+
@incoming_queue = val
|
52
|
+
end
|
53
|
+
|
54
|
+
def incoming_queue
|
55
|
+
@incoming_queue ||= "bus_incoming"
|
56
|
+
end
|
57
|
+
|
58
|
+
def hostname
|
59
|
+
@hostname ||= `hostname 2>&1`.strip.sub(/.local/,'')
|
60
|
+
end
|
61
|
+
|
62
|
+
def hostname=val
|
63
|
+
@hostname = val
|
64
|
+
end
|
65
|
+
|
66
|
+
def before_publish=(proc)
|
67
|
+
@before_publish_callback = proc
|
68
|
+
end
|
69
|
+
|
70
|
+
def before_publish_callback(attributes)
|
71
|
+
if @before_publish_callback
|
72
|
+
@before_publish_callback.call(attributes)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def logger
|
77
|
+
@logger
|
78
|
+
end
|
79
|
+
|
80
|
+
def logger=val
|
81
|
+
@logger = val
|
82
|
+
end
|
83
|
+
|
84
|
+
def log_application(message)
|
85
|
+
if logger
|
86
|
+
time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
|
87
|
+
logger.info("** [#{time}] #$$: QueueBus #{message}")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def log_worker(message)
|
92
|
+
if ENV['LOGGING'] || ENV['VERBOSE'] || ENV['VVERBOSE']
|
93
|
+
time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
|
94
|
+
puts "** [#{time}] #$$: #{message}"
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|