resque-bus 0.3.2 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +1 -0
  4. data/CHANGELOG.md +15 -0
  5. data/Gemfile +2 -3
  6. data/README.mdown +50 -64
  7. data/Rakefile +0 -1
  8. data/lib/resque-bus.rb +21 -283
  9. data/lib/resque_bus/adapter.rb +66 -0
  10. data/lib/resque_bus/compatibility/deprecated.rb +38 -0
  11. data/lib/resque_bus/compatibility/driver.rb +10 -0
  12. data/lib/resque_bus/compatibility/heartbeat.rb +10 -0
  13. data/lib/resque_bus/compatibility/publisher.rb +13 -0
  14. data/lib/resque_bus/compatibility/rider.rb +32 -0
  15. data/lib/resque_bus/compatibility/subscriber.rb +8 -0
  16. data/lib/resque_bus/compatibility/task_manager.rb +8 -0
  17. data/lib/resque_bus/server.rb +6 -5
  18. data/lib/resque_bus/server/views/bus.erb +2 -2
  19. data/lib/resque_bus/tasks.rb +46 -46
  20. data/lib/resque_bus/version.rb +2 -4
  21. data/resque-bus.gemspec +5 -12
  22. data/spec/adapter/compatibility_spec.rb +97 -0
  23. data/spec/adapter/integration_spec.rb +111 -0
  24. data/spec/adapter/publish_at_spec.rb +50 -0
  25. data/spec/adapter/retry_spec.rb +47 -0
  26. data/spec/adapter/support.rb +23 -0
  27. data/spec/adapter_spec.rb +14 -0
  28. data/spec/application_spec.rb +62 -62
  29. data/spec/config_spec.rb +83 -0
  30. data/spec/dispatch_spec.rb +6 -6
  31. data/spec/driver_spec.rb +62 -53
  32. data/spec/heartbeat_spec.rb +4 -4
  33. data/spec/integration_spec.rb +2 -2
  34. data/spec/matcher_spec.rb +29 -29
  35. data/spec/publish_spec.rb +62 -38
  36. data/spec/publisher_spec.rb +7 -0
  37. data/spec/rider_spec.rb +14 -66
  38. data/spec/spec_helper.rb +25 -28
  39. data/spec/subscriber_spec.rb +194 -176
  40. data/spec/subscription_list_spec.rb +1 -1
  41. data/spec/subscription_spec.rb +1 -1
  42. data/spec/worker_spec.rb +32 -0
  43. metadata +75 -91
  44. data/lib/resque_bus/application.rb +0 -115
  45. data/lib/resque_bus/dispatch.rb +0 -61
  46. data/lib/resque_bus/driver.rb +0 -30
  47. data/lib/resque_bus/heartbeat.rb +0 -106
  48. data/lib/resque_bus/local.rb +0 -34
  49. data/lib/resque_bus/matcher.rb +0 -81
  50. data/lib/resque_bus/publisher.rb +0 -12
  51. data/lib/resque_bus/rider.rb +0 -54
  52. data/lib/resque_bus/subscriber.rb +0 -63
  53. data/lib/resque_bus/subscription.rb +0 -55
  54. data/lib/resque_bus/subscription_list.rb +0 -53
  55. data/lib/resque_bus/task_manager.rb +0 -52
  56. data/lib/resque_bus/util.rb +0 -42
  57. data/lib/tasks/resquebus.rake +0 -2
  58. data/spec/publish_at_spec.rb +0 -74
  59. data/spec/redis_spec.rb +0 -13
@@ -1,61 +0,0 @@
1
- # Creates a DSL for apps to define their blocks to run for event_types
2
-
3
- module ResqueBus
4
- class Dispatch
5
-
6
- attr_reader :app_key, :subscriptions
7
-
8
- def initialize(app_key)
9
- @app_key = Application.normalize(app_key)
10
- @subscriptions = SubscriptionList.new
11
- end
12
-
13
- def size
14
- @subscriptions.size
15
- end
16
-
17
- def subscribe(key, matcher_hash = nil, &block)
18
- dispatch_event("default", key, matcher_hash, block)
19
- end
20
-
21
- # allows definitions of other queues
22
- def method_missing(method_name, *args, &block)
23
- if args.size == 1 && block
24
- dispatch_event(method_name, args[0], nil, block)
25
- elsif args.size == 2 && block
26
- dispatch_event(method_name, args[0], args[1], block)
27
- else
28
- super
29
- end
30
- end
31
-
32
- def execute(key, attributes)
33
- sub = subscriptions.key(key)
34
- if sub
35
- sub.execute!(attributes)
36
- else
37
- # TODO: log that it's not there
38
- end
39
- end
40
-
41
- def subscription_matches(attributes)
42
- out = subscriptions.matches(attributes)
43
- out.each do |sub|
44
- sub.app_key = self.app_key
45
- end
46
- out
47
- end
48
-
49
- def dispatch_event(queue, key, matcher_hash, block)
50
- # if not matcher_hash, assume key is a event_type regex
51
- matcher_hash ||= { "bus_event_type" => key }
52
- add_subscription("#{app_key}_#{queue}", key, "::ResqueBus::Rider", matcher_hash, block)
53
- end
54
-
55
- def add_subscription(queue_name, key, class_name, matcher_hash = nil, block)
56
- sub = Subscription.register(queue_name, key, class_name, matcher_hash, block)
57
- subscriptions.add(sub)
58
- sub
59
- end
60
- end
61
- end
@@ -1,30 +0,0 @@
1
- module ResqueBus
2
- # fans out an event to multiple queues
3
- class Driver
4
-
5
- class << self
6
- def subscription_matches(attributes)
7
- out = []
8
- Application.all.each do |app|
9
- subs = app.subscription_matches(attributes)
10
- out.concat(subs)
11
- end
12
- out
13
- end
14
-
15
- def perform(attributes={})
16
- raise "No attributes passed" if attributes.empty?
17
-
18
- ResqueBus.log_worker("Driver running: #{attributes.inspect}")
19
-
20
- subscription_matches(attributes).each do |sub|
21
- ResqueBus.log_worker(" ...sending to #{sub.queue_name} queue with class #{sub.class_name} for app #{sub.app_key} because of subscription: #{sub.key}")
22
-
23
- bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
24
- ResqueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
25
- end
26
- end
27
- end
28
-
29
- end
30
- end
@@ -1,106 +0,0 @@
1
- module ResqueBus
2
- # publishes event about the current time
3
- class Heartbeat
4
-
5
- class << self
6
-
7
- def lock_key
8
- "resquebus:heartbeat:lock"
9
- end
10
-
11
- def lock_seconds
12
- 60
13
- end
14
-
15
- def lock!
16
- now = Time.now.to_i
17
- timeout = now + lock_seconds + 2
18
-
19
- # return true if we successfully acquired the lock
20
- return timeout if Resque.redis.setnx(lock_key, timeout)
21
-
22
- # see if the existing timeout is still valid and return false if it is
23
- # (we cannot acquire the lock during the timeout period)
24
- return 0 if now <= Resque.redis.get(lock_key).to_i
25
-
26
- # otherwise set the timeout and ensure that no other worker has
27
- # acquired the lock
28
- if now > Resque.redis.getset(lock_key, timeout).to_i
29
- return timeout
30
- else
31
- return 0
32
- end
33
- end
34
-
35
- def unlock!
36
- Resque.redis.del(lock_key)
37
- end
38
-
39
-
40
- def redis_key
41
- "resquebus:heartbeat:timestamp"
42
- end
43
-
44
- def environment_name
45
- ENV["RAILS_ENV"] || ENV["RACK_ENV"] || ENV["RESQUEBUS_ENV"]
46
- end
47
-
48
- def get_saved_minute!
49
- key = ResqueBus.redis.get(redis_key)
50
- return nil if key.nil?
51
- case environment_name
52
- when 'development', 'test'
53
- # only 3 minutes in development; otherwise, TONS of events if not run in a while
54
- three_ago = Time.now.to_i - 3*60*60
55
- key = three_ago if key.to_i < three_ago
56
- end
57
- return key.to_i
58
- end
59
-
60
- def set_saved_minute!(epoch_minute)
61
- ResqueBus.redis.set(redis_key, epoch_minute)
62
- end
63
-
64
- def perform
65
- real_now = Time.now.to_i
66
- run_until = lock! - 2
67
- return if run_until < real_now
68
-
69
- while((real_now = Time.now.to_i) < run_until)
70
- minutes = real_now.to_i / 60
71
- last = get_saved_minute!
72
- if last
73
- break if minutes <= last
74
- minutes = last + 1
75
- end
76
-
77
- seconds = minutes * (60)
78
- hours = minutes / (60)
79
- days = minutes / (60*24)
80
-
81
- now = Time.at(seconds)
82
-
83
- attributes = {}
84
- attributes["epoch_seconds"] = seconds
85
- attributes["epoch_minutes"] = minutes
86
- attributes["epoch_hours"] = hours
87
- attributes["epoch_days"] = days
88
-
89
- attributes["minute"] = now.min
90
- attributes["hour"] = now.hour
91
- attributes["day"] = now.day
92
- attributes["month"] = now.month
93
- attributes["year"] = now.year
94
- attributes["yday"] = now.yday
95
- attributes["wday"] = now.wday
96
-
97
- ResqueBus.publish("heartbeat_minutes", attributes)
98
- set_saved_minute!(minutes)
99
- end
100
-
101
- unlock!
102
- end
103
- end
104
-
105
- end
106
- end
@@ -1,34 +0,0 @@
1
- module ResqueBus
2
- # only process local queues
3
- class Local
4
-
5
- class << self
6
- def perform(attributes = {})
7
- ResqueBus.log_worker("Local running: #{attributes.inspect}")
8
-
9
- # looking for subscriptions, not queues
10
- subscription_matches(attributes).each do |sub|
11
- bus_attr = {"bus_driven_at" => Time.now.to_i, "bus_rider_queue" => sub.queue_name, "bus_rider_app_key" => sub.app_key, "bus_rider_sub_key" => sub.key, "bus_rider_class_name" => sub.class_name}
12
- to_publish = bus_attr.merge(attributes || {})
13
- if ResqueBus.local_mode == :standalone
14
- ResqueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
15
- # defaults to inline mode
16
- else ResqueBus.local_mode == :inline
17
- sub.execute!(to_publish)
18
- end
19
- end
20
- end
21
-
22
- # looking directly at subscriptions loaded into dispatcher
23
- # so we don't need redis server up
24
- def subscription_matches(attributes)
25
- out = []
26
- ResqueBus.dispatchers.each do |dispatcher|
27
- out.concat(dispatcher.subscription_matches(attributes))
28
- end
29
- out
30
- end
31
- end
32
-
33
- end
34
- end
@@ -1,81 +0,0 @@
1
- module ResqueBus
2
- class Matcher
3
- SPECIAL_PREPEND = "bus_special_value_"
4
- attr_reader :filters
5
- def initialize(hash)
6
- @filters = encode(hash)
7
- end
8
-
9
- def to_redis
10
- @filters
11
- end
12
-
13
- def match?(attribute_name, attributes)
14
- mine = filters[attribute_name].to_s
15
- return false if mine.size == 0
16
-
17
- given = attributes[attribute_name]
18
- case mine
19
- when "#{SPECIAL_PREPEND}key"
20
- return true if attributes.has_key?(attribute_name)
21
- return false
22
- when "#{SPECIAL_PREPEND}blank"
23
- return true if given.to_s.strip.size == 0
24
- return false
25
- when "#{SPECIAL_PREPEND}empty"
26
- return false if given == nil
27
- return true if given.to_s.size == 0
28
- return false
29
- when "#{SPECIAL_PREPEND}nil"
30
- return true if given == nil
31
- return false
32
- when "#{SPECIAL_PREPEND}value"
33
- return false if given == nil
34
- return true
35
- when "#{SPECIAL_PREPEND}present"
36
- return true if given.to_s.strip.size > 0
37
- return false
38
- end
39
-
40
- given = given.to_s
41
-
42
- return true if mine == given
43
- begin
44
- # if it's already a regex, don't mess with it
45
- # otherwise, it should have start and end line situation
46
- if mine[0..6] == "(?-mix:"
47
- regex = Regexp.new(mine)
48
- else
49
- regex = Regexp.new("^#{mine}$")
50
- end
51
- return !!regex.match(given)
52
- rescue
53
- return false
54
- end
55
- end
56
-
57
- def matches?(attributes)
58
- return false if filters.empty?
59
- return false if attributes == nil
60
-
61
- filters.keys.each do |key|
62
- return false unless match?(key, attributes)
63
- end
64
-
65
- true
66
- end
67
-
68
- def encode(hash)
69
- out = {}
70
- hash.each do |key, value|
71
- case value
72
- when :key, :blank, :nil, :present, :empty, :value
73
- value = "#{SPECIAL_PREPEND}#{value}"
74
- end
75
- out[key.to_s] = value.to_s
76
- end
77
- out
78
- end
79
- end
80
- end
81
-
@@ -1,12 +0,0 @@
1
- module ResqueBus
2
- # publishes on a delay
3
- class Publisher
4
- class << self
5
- def perform(event_type, attributes = {})
6
- ResqueBus.log_worker("Publisher running: #{event_type} - #{attributes.inspect}")
7
- ResqueBus.publish(event_type, attributes)
8
- end
9
- end
10
-
11
- end
12
- end
@@ -1,54 +0,0 @@
1
- require 'resque-retry'
2
-
3
- module ResqueBus
4
- # queue'd in each
5
- class Rider
6
- extend Resque::Plugins::ExponentialBackoff
7
-
8
- class << self
9
- def perform(attributes = {})
10
- sub_key = attributes["bus_rider_sub_key"]
11
- app_key = attributes["bus_rider_app_key"]
12
- raise "No application key passed" if app_key.to_s == ""
13
- raise "No subcription key passed" if sub_key.to_s == ""
14
-
15
- attributes ||= {}
16
-
17
- ResqueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
18
-
19
- # attributes that should be available
20
- # attributes["bus_event_type"]
21
- # attributes["bus_app_key"]
22
- # attributes["bus_published_at"]
23
- # attributes["bus_driven_at"]
24
-
25
- # allow the real Reqsue to be used inside the callback while in a worker
26
- Resque.redis = ResqueBus.original_redis if ResqueBus.original_redis
27
-
28
- # (now running with the real app that subscribed)
29
- ResqueBus.dispatcher_execute(app_key, sub_key, attributes.merge("bus_executed_at" => Time.now.to_i))
30
- ensure
31
- # put this back if running in the thread
32
- Resque.redis = ResqueBus.redis if ResqueBus.original_redis
33
- end
34
-
35
- # @failure_hooks_already_ran on https://github.com/defunkt/resque/tree/1-x-stable
36
- # to prevent running twice
37
- def queue
38
- @my_queue
39
- end
40
-
41
- def on_failure_aaa(exception, *args)
42
- # note: sorted alphabetically
43
- # queue needs to be set for rety to work (know what queue in Requeue.class_to_queue)
44
- @my_queue = args[0]["bus_rider_queue"]
45
- end
46
-
47
- def on_failure_zzz(exception, *args)
48
- # note: sorted alphabetically
49
- @my_queue = nil
50
- end
51
-
52
- end
53
- end
54
- end
@@ -1,63 +0,0 @@
1
- module ResqueBus
2
- module Subscriber
3
-
4
- def self.included(base)
5
- base.extend ClassMethods
6
- end
7
-
8
- module ClassMethods
9
-
10
- def application(app_key)
11
- @app_key = ::ResqueBus::Application.normalize(app_key)
12
- end
13
-
14
- def app_key
15
- return @app_key if @app_key
16
- @app_key = ::ResqueBus.default_app_key
17
- return @app_key if @app_key
18
- # module or class_name
19
- val = self.name.to_s.split("::").first
20
- @app_key = ::ResqueBus::Util.underscore(val)
21
- end
22
-
23
- def subscribe(method_name, matcher_hash = nil)
24
- queue_name = ::Resque.queue_from_class(self)
25
- queue_name ||= ::ResqueBus.default_queue
26
- queue_name ||= "#{app_key}_default"
27
- subscribe_queue(queue_name, method_name, matcher_hash)
28
- end
29
-
30
- def subscribe_queue(queue_name, method_name, matcher_hash = nil)
31
- klass = self
32
- matcher_hash ||= {"bus_event_type" => method_name}
33
- sub_key = "#{self.name}.#{method_name}"
34
- dispatcher = ::ResqueBus.dispatcher_by_key(app_key)
35
- dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash, lambda{ |att| klass.perform(att) })
36
- end
37
-
38
- def transform(method_name)
39
- @transform = method_name
40
- end
41
-
42
- def perform(attributes)
43
- ResqueBus.with_global_attributes(attributes) do
44
- sub_key = attributes["bus_rider_sub_key"]
45
- meth_key = sub_key.split(".").last
46
- resque_bus_execute(meth_key, attributes)
47
- end
48
- end
49
-
50
- def resque_bus_execute(key, attributes)
51
- args = attributes
52
- args = send(@transform, attributes) if @transform
53
- args = [args] unless args.is_a?(Array)
54
- if self.respond_to?(:subscriber_with_attributes)
55
- me = self.subscriber_with_attributes(attributes)
56
- else
57
- me = self.new
58
- end
59
- me.send(key, *args)
60
- end
61
- end
62
- end
63
- end