resque-bus 0.3.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.rspec +1 -0
- data/CHANGELOG.md +15 -0
- data/Gemfile +2 -3
- data/README.mdown +50 -64
- data/Rakefile +0 -1
- data/lib/resque-bus.rb +21 -283
- data/lib/resque_bus/adapter.rb +66 -0
- data/lib/resque_bus/compatibility/deprecated.rb +38 -0
- data/lib/resque_bus/compatibility/driver.rb +10 -0
- data/lib/resque_bus/compatibility/heartbeat.rb +10 -0
- data/lib/resque_bus/compatibility/publisher.rb +13 -0
- data/lib/resque_bus/compatibility/rider.rb +32 -0
- data/lib/resque_bus/compatibility/subscriber.rb +8 -0
- data/lib/resque_bus/compatibility/task_manager.rb +8 -0
- data/lib/resque_bus/server.rb +6 -5
- data/lib/resque_bus/server/views/bus.erb +2 -2
- data/lib/resque_bus/tasks.rb +46 -46
- data/lib/resque_bus/version.rb +2 -4
- data/resque-bus.gemspec +5 -12
- data/spec/adapter/compatibility_spec.rb +97 -0
- data/spec/adapter/integration_spec.rb +111 -0
- data/spec/adapter/publish_at_spec.rb +50 -0
- data/spec/adapter/retry_spec.rb +47 -0
- data/spec/adapter/support.rb +23 -0
- data/spec/adapter_spec.rb +14 -0
- data/spec/application_spec.rb +62 -62
- data/spec/config_spec.rb +83 -0
- data/spec/dispatch_spec.rb +6 -6
- data/spec/driver_spec.rb +62 -53
- data/spec/heartbeat_spec.rb +4 -4
- data/spec/integration_spec.rb +2 -2
- data/spec/matcher_spec.rb +29 -29
- data/spec/publish_spec.rb +62 -38
- data/spec/publisher_spec.rb +7 -0
- data/spec/rider_spec.rb +14 -66
- data/spec/spec_helper.rb +25 -28
- data/spec/subscriber_spec.rb +194 -176
- data/spec/subscription_list_spec.rb +1 -1
- data/spec/subscription_spec.rb +1 -1
- data/spec/worker_spec.rb +32 -0
- metadata +75 -91
- data/lib/resque_bus/application.rb +0 -115
- data/lib/resque_bus/dispatch.rb +0 -61
- data/lib/resque_bus/driver.rb +0 -30
- data/lib/resque_bus/heartbeat.rb +0 -106
- data/lib/resque_bus/local.rb +0 -34
- data/lib/resque_bus/matcher.rb +0 -81
- data/lib/resque_bus/publisher.rb +0 -12
- data/lib/resque_bus/rider.rb +0 -54
- data/lib/resque_bus/subscriber.rb +0 -63
- data/lib/resque_bus/subscription.rb +0 -55
- data/lib/resque_bus/subscription_list.rb +0 -53
- data/lib/resque_bus/task_manager.rb +0 -52
- data/lib/resque_bus/util.rb +0 -42
- data/lib/tasks/resquebus.rake +0 -2
- data/spec/publish_at_spec.rb +0 -74
- data/spec/redis_spec.rb +0 -13
data/lib/resque_bus/dispatch.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
# Creates a DSL for apps to define their blocks to run for event_types
|
2
|
-
|
3
|
-
module ResqueBus
|
4
|
-
class Dispatch
|
5
|
-
|
6
|
-
attr_reader :app_key, :subscriptions
|
7
|
-
|
8
|
-
def initialize(app_key)
|
9
|
-
@app_key = Application.normalize(app_key)
|
10
|
-
@subscriptions = SubscriptionList.new
|
11
|
-
end
|
12
|
-
|
13
|
-
def size
|
14
|
-
@subscriptions.size
|
15
|
-
end
|
16
|
-
|
17
|
-
def subscribe(key, matcher_hash = nil, &block)
|
18
|
-
dispatch_event("default", key, matcher_hash, block)
|
19
|
-
end
|
20
|
-
|
21
|
-
# allows definitions of other queues
|
22
|
-
def method_missing(method_name, *args, &block)
|
23
|
-
if args.size == 1 && block
|
24
|
-
dispatch_event(method_name, args[0], nil, block)
|
25
|
-
elsif args.size == 2 && block
|
26
|
-
dispatch_event(method_name, args[0], args[1], block)
|
27
|
-
else
|
28
|
-
super
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def execute(key, attributes)
|
33
|
-
sub = subscriptions.key(key)
|
34
|
-
if sub
|
35
|
-
sub.execute!(attributes)
|
36
|
-
else
|
37
|
-
# TODO: log that it's not there
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
def subscription_matches(attributes)
|
42
|
-
out = subscriptions.matches(attributes)
|
43
|
-
out.each do |sub|
|
44
|
-
sub.app_key = self.app_key
|
45
|
-
end
|
46
|
-
out
|
47
|
-
end
|
48
|
-
|
49
|
-
def dispatch_event(queue, key, matcher_hash, block)
|
50
|
-
# if not matcher_hash, assume key is a event_type regex
|
51
|
-
matcher_hash ||= { "bus_event_type" => key }
|
52
|
-
add_subscription("#{app_key}_#{queue}", key, "::ResqueBus::Rider", matcher_hash, block)
|
53
|
-
end
|
54
|
-
|
55
|
-
def add_subscription(queue_name, key, class_name, matcher_hash = nil, block)
|
56
|
-
sub = Subscription.register(queue_name, key, class_name, matcher_hash, block)
|
57
|
-
subscriptions.add(sub)
|
58
|
-
sub
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
data/lib/resque_bus/driver.rb
DELETED
@@ -1,30 +0,0 @@
|
|
1
|
-
module ResqueBus
|
2
|
-
# fans out an event to multiple queues
|
3
|
-
class Driver
|
4
|
-
|
5
|
-
class << self
|
6
|
-
def subscription_matches(attributes)
|
7
|
-
out = []
|
8
|
-
Application.all.each do |app|
|
9
|
-
subs = app.subscription_matches(attributes)
|
10
|
-
out.concat(subs)
|
11
|
-
end
|
12
|
-
out
|
13
|
-
end
|
14
|
-
|
15
|
-
def perform(attributes={})
|
16
|
-
raise "No attributes passed" if attributes.empty?
|
17
|
-
|
18
|
-
ResqueBus.log_worker("Driver running: #{attributes.inspect}")
|
19
|
-
|
20
|
-
subscription_matches(attributes).each do |sub|
|
21
|
-
ResqueBus.log_worker(" ...sending to #{sub.queue_name} queue with class #{sub.class_name} for app #{sub.app_key} because of subscription: #{sub.key}")
|
22
|
-
|
23
|
-
bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
|
24
|
-
ResqueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
end
|
30
|
-
end
|
data/lib/resque_bus/heartbeat.rb
DELETED
@@ -1,106 +0,0 @@
|
|
1
|
-
module ResqueBus
|
2
|
-
# publishes event about the current time
|
3
|
-
class Heartbeat
|
4
|
-
|
5
|
-
class << self
|
6
|
-
|
7
|
-
def lock_key
|
8
|
-
"resquebus:heartbeat:lock"
|
9
|
-
end
|
10
|
-
|
11
|
-
def lock_seconds
|
12
|
-
60
|
13
|
-
end
|
14
|
-
|
15
|
-
def lock!
|
16
|
-
now = Time.now.to_i
|
17
|
-
timeout = now + lock_seconds + 2
|
18
|
-
|
19
|
-
# return true if we successfully acquired the lock
|
20
|
-
return timeout if Resque.redis.setnx(lock_key, timeout)
|
21
|
-
|
22
|
-
# see if the existing timeout is still valid and return false if it is
|
23
|
-
# (we cannot acquire the lock during the timeout period)
|
24
|
-
return 0 if now <= Resque.redis.get(lock_key).to_i
|
25
|
-
|
26
|
-
# otherwise set the timeout and ensure that no other worker has
|
27
|
-
# acquired the lock
|
28
|
-
if now > Resque.redis.getset(lock_key, timeout).to_i
|
29
|
-
return timeout
|
30
|
-
else
|
31
|
-
return 0
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
def unlock!
|
36
|
-
Resque.redis.del(lock_key)
|
37
|
-
end
|
38
|
-
|
39
|
-
|
40
|
-
def redis_key
|
41
|
-
"resquebus:heartbeat:timestamp"
|
42
|
-
end
|
43
|
-
|
44
|
-
def environment_name
|
45
|
-
ENV["RAILS_ENV"] || ENV["RACK_ENV"] || ENV["RESQUEBUS_ENV"]
|
46
|
-
end
|
47
|
-
|
48
|
-
def get_saved_minute!
|
49
|
-
key = ResqueBus.redis.get(redis_key)
|
50
|
-
return nil if key.nil?
|
51
|
-
case environment_name
|
52
|
-
when 'development', 'test'
|
53
|
-
# only 3 minutes in development; otherwise, TONS of events if not run in a while
|
54
|
-
three_ago = Time.now.to_i - 3*60*60
|
55
|
-
key = three_ago if key.to_i < three_ago
|
56
|
-
end
|
57
|
-
return key.to_i
|
58
|
-
end
|
59
|
-
|
60
|
-
def set_saved_minute!(epoch_minute)
|
61
|
-
ResqueBus.redis.set(redis_key, epoch_minute)
|
62
|
-
end
|
63
|
-
|
64
|
-
def perform
|
65
|
-
real_now = Time.now.to_i
|
66
|
-
run_until = lock! - 2
|
67
|
-
return if run_until < real_now
|
68
|
-
|
69
|
-
while((real_now = Time.now.to_i) < run_until)
|
70
|
-
minutes = real_now.to_i / 60
|
71
|
-
last = get_saved_minute!
|
72
|
-
if last
|
73
|
-
break if minutes <= last
|
74
|
-
minutes = last + 1
|
75
|
-
end
|
76
|
-
|
77
|
-
seconds = minutes * (60)
|
78
|
-
hours = minutes / (60)
|
79
|
-
days = minutes / (60*24)
|
80
|
-
|
81
|
-
now = Time.at(seconds)
|
82
|
-
|
83
|
-
attributes = {}
|
84
|
-
attributes["epoch_seconds"] = seconds
|
85
|
-
attributes["epoch_minutes"] = minutes
|
86
|
-
attributes["epoch_hours"] = hours
|
87
|
-
attributes["epoch_days"] = days
|
88
|
-
|
89
|
-
attributes["minute"] = now.min
|
90
|
-
attributes["hour"] = now.hour
|
91
|
-
attributes["day"] = now.day
|
92
|
-
attributes["month"] = now.month
|
93
|
-
attributes["year"] = now.year
|
94
|
-
attributes["yday"] = now.yday
|
95
|
-
attributes["wday"] = now.wday
|
96
|
-
|
97
|
-
ResqueBus.publish("heartbeat_minutes", attributes)
|
98
|
-
set_saved_minute!(minutes)
|
99
|
-
end
|
100
|
-
|
101
|
-
unlock!
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
end
|
106
|
-
end
|
data/lib/resque_bus/local.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
module ResqueBus
|
2
|
-
# only process local queues
|
3
|
-
class Local
|
4
|
-
|
5
|
-
class << self
|
6
|
-
def perform(attributes = {})
|
7
|
-
ResqueBus.log_worker("Local running: #{attributes.inspect}")
|
8
|
-
|
9
|
-
# looking for subscriptions, not queues
|
10
|
-
subscription_matches(attributes).each do |sub|
|
11
|
-
bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
|
12
|
-
to_publish = bus_attr.merge(attributes || {})
|
13
|
-
if ResqueBus.local_mode == :standalone
|
14
|
-
ResqueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
|
15
|
-
# defaults to inline mode
|
16
|
-
else ResqueBus.local_mode == :inline
|
17
|
-
sub.execute!(to_publish)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
# looking directly at subscriptions loaded into dispatcher
|
23
|
-
# so we don't need redis server up
|
24
|
-
def subscription_matches(attributes)
|
25
|
-
out = []
|
26
|
-
ResqueBus.dispatchers.each do |dispatcher|
|
27
|
-
out.concat(dispatcher.subscription_matches(attributes))
|
28
|
-
end
|
29
|
-
out
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
end
|
data/lib/resque_bus/matcher.rb
DELETED
@@ -1,81 +0,0 @@
|
|
1
|
-
module ResqueBus
|
2
|
-
class Matcher
|
3
|
-
SPECIAL_PREPEND = "bus_special_value_"
|
4
|
-
attr_reader :filters
|
5
|
-
def initialize(hash)
|
6
|
-
@filters = encode(hash)
|
7
|
-
end
|
8
|
-
|
9
|
-
def to_redis
|
10
|
-
@filters
|
11
|
-
end
|
12
|
-
|
13
|
-
def match?(attribute_name, attributes)
|
14
|
-
mine = filters[attribute_name].to_s
|
15
|
-
return false if mine.size == 0
|
16
|
-
|
17
|
-
given = attributes[attribute_name]
|
18
|
-
case mine
|
19
|
-
when "#{SPECIAL_PREPEND}key"
|
20
|
-
return true if attributes.has_key?(attribute_name)
|
21
|
-
return false
|
22
|
-
when "#{SPECIAL_PREPEND}blank"
|
23
|
-
return true if given.to_s.strip.size == 0
|
24
|
-
return false
|
25
|
-
when "#{SPECIAL_PREPEND}empty"
|
26
|
-
return false if given == nil
|
27
|
-
return true if given.to_s.size == 0
|
28
|
-
return false
|
29
|
-
when "#{SPECIAL_PREPEND}nil"
|
30
|
-
return true if given == nil
|
31
|
-
return false
|
32
|
-
when "#{SPECIAL_PREPEND}value"
|
33
|
-
return false if given == nil
|
34
|
-
return true
|
35
|
-
when "#{SPECIAL_PREPEND}present"
|
36
|
-
return true if given.to_s.strip.size > 0
|
37
|
-
return false
|
38
|
-
end
|
39
|
-
|
40
|
-
given = given.to_s
|
41
|
-
|
42
|
-
return true if mine == given
|
43
|
-
begin
|
44
|
-
# if it's already a regex, don't mess with it
|
45
|
-
# otherwise, it should have start and end line situation
|
46
|
-
if mine[0..6] == "(?-mix:"
|
47
|
-
regex = Regexp.new(mine)
|
48
|
-
else
|
49
|
-
regex = Regexp.new("^#{mine}$")
|
50
|
-
end
|
51
|
-
return !!regex.match(given)
|
52
|
-
rescue
|
53
|
-
return false
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
def matches?(attributes)
|
58
|
-
return false if filters.empty?
|
59
|
-
return false if attributes == nil
|
60
|
-
|
61
|
-
filters.keys.each do |key|
|
62
|
-
return false unless match?(key, attributes)
|
63
|
-
end
|
64
|
-
|
65
|
-
true
|
66
|
-
end
|
67
|
-
|
68
|
-
def encode(hash)
|
69
|
-
out = {}
|
70
|
-
hash.each do |key, value|
|
71
|
-
case value
|
72
|
-
when :key, :blank, :nil, :present, :empty, :value
|
73
|
-
value = "#{SPECIAL_PREPEND}#{value}"
|
74
|
-
end
|
75
|
-
out[key.to_s] = value.to_s
|
76
|
-
end
|
77
|
-
out
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
|
-
|
data/lib/resque_bus/publisher.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
module ResqueBus
|
2
|
-
# publishes on a delay
|
3
|
-
class Publisher
|
4
|
-
class << self
|
5
|
-
def perform(event_type, attributes = {})
|
6
|
-
ResqueBus.log_worker("Publisher running: #{event_type} - #{attributes.inspect}")
|
7
|
-
ResqueBus.publish(event_type, attributes)
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
end
|
12
|
-
end
|
data/lib/resque_bus/rider.rb
DELETED
@@ -1,54 +0,0 @@
|
|
1
|
-
require 'resque-retry'
|
2
|
-
|
3
|
-
module ResqueBus
|
4
|
-
# queue'd in each
|
5
|
-
class Rider
|
6
|
-
extend Resque::Plugins::ExponentialBackoff
|
7
|
-
|
8
|
-
class << self
|
9
|
-
def perform(attributes = {})
|
10
|
-
sub_key = attributes["bus_rider_sub_key"]
|
11
|
-
app_key = attributes["bus_rider_app_key"]
|
12
|
-
raise "No application key passed" if app_key.to_s == ""
|
13
|
-
raise "No subcription key passed" if sub_key.to_s == ""
|
14
|
-
|
15
|
-
attributes ||= {}
|
16
|
-
|
17
|
-
ResqueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
|
18
|
-
|
19
|
-
# attributes that should be available
|
20
|
-
# attributes["bus_event_type"]
|
21
|
-
# attributes["bus_app_key"]
|
22
|
-
# attributes["bus_published_at"]
|
23
|
-
# attributes["bus_driven_at"]
|
24
|
-
|
25
|
-
# allow the real Reqsue to be used inside the callback while in a worker
|
26
|
-
Resque.redis = ResqueBus.original_redis if ResqueBus.original_redis
|
27
|
-
|
28
|
-
# (now running with the real app that subscribed)
|
29
|
-
ResqueBus.dispatcher_execute(app_key, sub_key, attributes.merge("bus_executed_at" => Time.now.to_i))
|
30
|
-
ensure
|
31
|
-
# put this back if running in the thread
|
32
|
-
Resque.redis = ResqueBus.redis if ResqueBus.original_redis
|
33
|
-
end
|
34
|
-
|
35
|
-
# @failure_hooks_already_ran on https://github.com/defunkt/resque/tree/1-x-stable
|
36
|
-
# to prevent running twice
|
37
|
-
def queue
|
38
|
-
@my_queue
|
39
|
-
end
|
40
|
-
|
41
|
-
def on_failure_aaa(exception, *args)
|
42
|
-
# note: sorted alphabetically
|
43
|
-
# queue needs to be set for rety to work (know what queue in Requeue.class_to_queue)
|
44
|
-
@my_queue = args[0]["bus_rider_queue"]
|
45
|
-
end
|
46
|
-
|
47
|
-
def on_failure_zzz(exception, *args)
|
48
|
-
# note: sorted alphabetically
|
49
|
-
@my_queue = nil
|
50
|
-
end
|
51
|
-
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,63 +0,0 @@
|
|
1
|
-
module ResqueBus
|
2
|
-
module Subscriber
|
3
|
-
|
4
|
-
def self.included(base)
|
5
|
-
base.extend ClassMethods
|
6
|
-
end
|
7
|
-
|
8
|
-
module ClassMethods
|
9
|
-
|
10
|
-
def application(app_key)
|
11
|
-
@app_key = ::ResqueBus::Application.normalize(app_key)
|
12
|
-
end
|
13
|
-
|
14
|
-
def app_key
|
15
|
-
return @app_key if @app_key
|
16
|
-
@app_key = ::ResqueBus.default_app_key
|
17
|
-
return @app_key if @app_key
|
18
|
-
# module or class_name
|
19
|
-
val = self.name.to_s.split("::").first
|
20
|
-
@app_key = ::ResqueBus::Util.underscore(val)
|
21
|
-
end
|
22
|
-
|
23
|
-
def subscribe(method_name, matcher_hash = nil)
|
24
|
-
queue_name = ::Resque.queue_from_class(self)
|
25
|
-
queue_name ||= ::ResqueBus.default_queue
|
26
|
-
queue_name ||= "#{app_key}_default"
|
27
|
-
subscribe_queue(queue_name, method_name, matcher_hash)
|
28
|
-
end
|
29
|
-
|
30
|
-
def subscribe_queue(queue_name, method_name, matcher_hash = nil)
|
31
|
-
klass = self
|
32
|
-
matcher_hash ||= {"bus_event_type" => method_name}
|
33
|
-
sub_key = "#{self.name}.#{method_name}"
|
34
|
-
dispatcher = ::ResqueBus.dispatcher_by_key(app_key)
|
35
|
-
dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash, lambda{ |att| klass.perform(att) })
|
36
|
-
end
|
37
|
-
|
38
|
-
def transform(method_name)
|
39
|
-
@transform = method_name
|
40
|
-
end
|
41
|
-
|
42
|
-
def perform(attributes)
|
43
|
-
ResqueBus.with_global_attributes(attributes) do
|
44
|
-
sub_key = attributes["bus_rider_sub_key"]
|
45
|
-
meth_key = sub_key.split(".").last
|
46
|
-
resque_bus_execute(meth_key, attributes)
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def resque_bus_execute(key, attributes)
|
51
|
-
args = attributes
|
52
|
-
args = send(@transform, attributes) if @transform
|
53
|
-
args = [args] unless args.is_a?(Array)
|
54
|
-
if self.respond_to?(:subscriber_with_attributes)
|
55
|
-
me = self.subscriber_with_attributes(attributes)
|
56
|
-
else
|
57
|
-
me = self.new
|
58
|
-
end
|
59
|
-
me.send(key, *args)
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|