queue-bus 0.9.0 → 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/CHANGELOG.md +9 -0
- data/Gemfile +4 -2
- data/Rakefile +2 -0
- data/lib/queue-bus.rb +15 -12
- data/lib/queue_bus/adapters/base.rb +4 -2
- data/lib/queue_bus/adapters/data.rb +12 -11
- data/lib/queue_bus/application.rb +13 -15
- data/lib/queue_bus/dispatch.rb +14 -12
- data/lib/queue_bus/dispatchers.rb +12 -5
- data/lib/queue_bus/driver.rb +15 -10
- data/lib/queue_bus/heartbeat.rb +32 -30
- data/lib/queue_bus/local.rb +9 -9
- data/lib/queue_bus/matcher.rb +36 -27
- data/lib/queue_bus/publisher.rb +7 -5
- data/lib/queue_bus/publishing.rb +31 -24
- data/lib/queue_bus/rider.rb +26 -22
- data/lib/queue_bus/subscriber.rb +20 -14
- data/lib/queue_bus/subscription.rb +25 -15
- data/lib/queue_bus/subscription_list.rb +30 -12
- data/lib/queue_bus/task_manager.rb +18 -16
- data/lib/queue_bus/tasks.rb +20 -15
- data/lib/queue_bus/util.rb +11 -8
- data/lib/queue_bus/version.rb +3 -1
- data/lib/queue_bus/worker.rb +3 -2
- data/queue-bus.gemspec +19 -18
- data/spec/adapter/publish_at_spec.rb +28 -25
- data/spec/adapter/support.rb +7 -1
- data/spec/adapter_spec.rb +4 -2
- data/spec/application_spec.rb +97 -97
- data/spec/dispatch_spec.rb +48 -51
- data/spec/driver_spec.rb +60 -58
- data/spec/heartbeat_spec.rb +26 -24
- data/spec/integration_spec.rb +41 -40
- data/spec/matcher_spec.rb +104 -102
- data/spec/publish_spec.rb +46 -46
- data/spec/publisher_spec.rb +3 -1
- data/spec/rider_spec.rb +16 -14
- data/spec/spec_helper.rb +2 -2
- data/spec/subscriber_spec.rb +227 -227
- data/spec/subscription_list_spec.rb +31 -31
- data/spec/subscription_spec.rb +37 -36
- data/spec/worker_spec.rb +17 -15
- metadata +7 -6
data/lib/queue_bus/subscriber.rb
CHANGED
@@ -1,28 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A mixin to configure subscriptions on a particular class
|
2
5
|
module Subscriber
|
3
|
-
|
4
6
|
def self.included(base)
|
5
7
|
base.extend ClassMethods
|
6
8
|
end
|
7
9
|
|
10
|
+
# The class methods that will be added to the class it's included in. Use them to
|
11
|
+
# configure and subscribe.
|
8
12
|
module ClassMethods
|
9
|
-
|
10
13
|
def application(app_key)
|
11
14
|
@app_key = ::QueueBus::Application.normalize(app_key)
|
12
15
|
end
|
13
16
|
|
14
17
|
def app_key
|
15
18
|
return @app_key if @app_key
|
19
|
+
|
16
20
|
@app_key = ::QueueBus.default_app_key
|
17
21
|
return @app_key if @app_key
|
22
|
+
|
18
23
|
# module or class_name
|
19
|
-
val =
|
24
|
+
val = name.to_s.split('::').first
|
20
25
|
@app_key = ::QueueBus::Util.underscore(val)
|
21
26
|
end
|
22
27
|
|
23
28
|
def subscribe(method_name, matcher_hash = nil)
|
24
29
|
queue_name = nil
|
25
|
-
queue_name ||=
|
30
|
+
queue_name ||= instance_variable_get(:@queue) || (respond_to?(:queue) && queue)
|
26
31
|
queue_name ||= ::QueueBus.default_queue
|
27
32
|
queue_name ||= "#{app_key}_default"
|
28
33
|
subscribe_queue(queue_name, method_name, matcher_hash)
|
@@ -30,10 +35,11 @@ module QueueBus
|
|
30
35
|
|
31
36
|
def subscribe_queue(queue_name, method_name, matcher_hash = nil)
|
32
37
|
klass = self
|
33
|
-
matcher_hash ||= {
|
34
|
-
sub_key = "#{
|
38
|
+
matcher_hash ||= { 'bus_event_type' => method_name }
|
39
|
+
sub_key = "#{name}.#{method_name}"
|
35
40
|
dispatcher = ::QueueBus.dispatcher_by_key(app_key)
|
36
|
-
dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash,
|
41
|
+
dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash,
|
42
|
+
->(att) { klass.perform(att) })
|
37
43
|
end
|
38
44
|
|
39
45
|
def transform(method_name)
|
@@ -42,8 +48,8 @@ module QueueBus
|
|
42
48
|
|
43
49
|
def perform(attributes)
|
44
50
|
::QueueBus.with_global_attributes(attributes) do
|
45
|
-
sub_key = attributes[
|
46
|
-
meth_key = sub_key.split(
|
51
|
+
sub_key = attributes['bus_rider_sub_key']
|
52
|
+
meth_key = sub_key.split('.').last
|
47
53
|
queue_bus_execute(meth_key, attributes)
|
48
54
|
end
|
49
55
|
end
|
@@ -52,11 +58,11 @@ module QueueBus
|
|
52
58
|
args = attributes
|
53
59
|
args = send(@transform, attributes) if @transform
|
54
60
|
args = [args] unless args.is_a?(Array)
|
55
|
-
if
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
61
|
+
me = if respond_to?(:subscriber_with_attributes)
|
62
|
+
subscriber_with_attributes(attributes)
|
63
|
+
else
|
64
|
+
new
|
65
|
+
end
|
60
66
|
me.send(key, *args)
|
61
67
|
end
|
62
68
|
end
|
@@ -1,29 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A Subscription is the destination of an event.
|
5
|
+
#
|
6
|
+
# The subscription can be stored in redis but should only be executed on ruby processes that
|
7
|
+
# have the application loaded. In general, this is controlled by having the background workers
|
8
|
+
# listen to specific (and discrete) queues.
|
2
9
|
class Subscription
|
3
|
-
|
4
10
|
class << self
|
5
11
|
def register(queue, key, class_name, matcher, block)
|
6
12
|
Subscription.new(queue, key, class_name, matcher, block)
|
7
13
|
end
|
8
14
|
|
9
15
|
def from_redis(hash)
|
10
|
-
queue_name = hash[
|
11
|
-
key = hash[
|
12
|
-
class_name = hash[
|
13
|
-
matcher = hash[
|
14
|
-
return nil if key.
|
16
|
+
queue_name = hash['queue_name'].to_s
|
17
|
+
key = hash['key'].to_s
|
18
|
+
class_name = hash['class'].to_s
|
19
|
+
matcher = hash['matcher']
|
20
|
+
return nil if key.empty? || queue_name.empty?
|
21
|
+
|
15
22
|
Subscription.new(queue_name, key, class_name, matcher, nil)
|
16
23
|
end
|
17
24
|
|
18
25
|
def normalize(val)
|
19
|
-
val.to_s.gsub(/\W/,
|
26
|
+
val.to_s.gsub(/\W/, '_').downcase
|
20
27
|
end
|
21
28
|
end
|
22
29
|
|
23
30
|
attr_reader :matcher, :executor, :queue_name, :key, :class_name
|
24
|
-
attr_accessor :app_key
|
31
|
+
attr_accessor :app_key # dyanmically set on return from subscription_matches
|
25
32
|
|
26
|
-
def initialize(queue_name, key, class_name, filters, executor=nil)
|
33
|
+
def initialize(queue_name, key, class_name, filters, executor = nil)
|
27
34
|
@queue_name = self.class.normalize(queue_name)
|
28
35
|
@key = key.to_s
|
29
36
|
@class_name = class_name.to_s
|
@@ -31,8 +38,12 @@ module QueueBus
|
|
31
38
|
@executor = executor
|
32
39
|
end
|
33
40
|
|
41
|
+
# Executes the subscription. If this is run on a server/ruby process that did not subscribe
|
42
|
+
# it will error as there will not be a proc.
|
34
43
|
def execute!(attributes)
|
35
|
-
|
44
|
+
if attributes.respond_to?(:with_indifferent_access)
|
45
|
+
attributes = attributes.with_indifferent_access
|
46
|
+
end
|
36
47
|
::QueueBus.with_global_attributes(attributes) do
|
37
48
|
executor.call(attributes)
|
38
49
|
end
|
@@ -44,12 +55,11 @@ module QueueBus
|
|
44
55
|
|
45
56
|
def to_redis
|
46
57
|
out = {}
|
47
|
-
out[
|
48
|
-
out[
|
49
|
-
out[
|
50
|
-
out[
|
58
|
+
out['queue_name'] = queue_name
|
59
|
+
out['key'] = key
|
60
|
+
out['class'] = class_name
|
61
|
+
out['matcher'] = matcher.to_redis
|
51
62
|
out
|
52
63
|
end
|
53
|
-
|
54
64
|
end
|
55
65
|
end
|
@@ -1,11 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# Manages a set of subscriptions.
|
5
|
+
#
|
6
|
+
# The subscriptions are stored in redis but not by this class. Instead this class uses two
|
7
|
+
# functions `to_redis` and `from_redis` to facilitate serialization without accessing redis
|
8
|
+
# directly.
|
9
|
+
#
|
10
|
+
# To create a new SubscriptionList, use the static function `from_redis` and pass
|
11
|
+
# it a hash that came from redis.
|
12
|
+
#
|
13
|
+
# To get a value fro redis, take your loaded SubscriptionList and call `to_redis` on it. The
|
14
|
+
# returned value can be used to store in redis.
|
2
15
|
class SubscriptionList
|
3
|
-
|
4
16
|
class << self
|
5
17
|
def from_redis(redis_hash)
|
6
18
|
out = SubscriptionList.new
|
7
|
-
|
8
|
-
redis_hash.each do |
|
19
|
+
|
20
|
+
redis_hash.each do |_key, value|
|
9
21
|
sub = Subscription.from_redis(value)
|
10
22
|
out.add(sub) if sub
|
11
23
|
end
|
@@ -21,29 +33,35 @@ module QueueBus
|
|
21
33
|
end
|
22
34
|
out
|
23
35
|
end
|
24
|
-
|
36
|
+
|
25
37
|
def initialize
|
26
38
|
@subscriptions = {}
|
27
39
|
end
|
28
|
-
|
40
|
+
|
29
41
|
def add(sub)
|
30
|
-
|
31
|
-
|
42
|
+
if @subscriptions.key?(sub.key)
|
43
|
+
raise "Duplicate key: #{sub.key} already exists " \
|
44
|
+
"in the #{sub.queue_name} queue!"
|
45
|
+
end
|
32
46
|
@subscriptions[sub.key] = sub
|
33
47
|
end
|
34
|
-
|
48
|
+
|
35
49
|
def size
|
36
50
|
@subscriptions.size
|
37
51
|
end
|
38
|
-
|
52
|
+
|
53
|
+
def empty?
|
54
|
+
size.zero?
|
55
|
+
end
|
56
|
+
|
39
57
|
def key(key)
|
40
58
|
@subscriptions[key.to_s]
|
41
59
|
end
|
42
|
-
|
60
|
+
|
43
61
|
def all
|
44
62
|
@subscriptions.values
|
45
63
|
end
|
46
|
-
|
64
|
+
|
47
65
|
def matches(attributes)
|
48
66
|
out = []
|
49
67
|
all.each do |sub|
|
@@ -52,4 +70,4 @@ module QueueBus
|
|
52
70
|
out
|
53
71
|
end
|
54
72
|
end
|
55
|
-
end
|
73
|
+
end
|
@@ -1,27 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A helper class for executing Rake tasks.
|
2
5
|
class TaskManager
|
3
|
-
|
4
6
|
attr_reader :logging
|
5
|
-
|
7
|
+
|
6
8
|
def initialize(logging)
|
7
9
|
@logging = logging
|
8
10
|
end
|
9
|
-
|
11
|
+
|
10
12
|
def subscribe!
|
11
13
|
count = 0
|
12
14
|
::QueueBus.dispatchers.each do |dispatcher|
|
13
15
|
subscriptions = dispatcher.subscriptions
|
14
|
-
if subscriptions.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
next if subscriptions.empty?
|
17
|
+
|
18
|
+
count += subscriptions.size
|
19
|
+
log "Subscribing #{dispatcher.app_key} to #{subscriptions.size} subscriptions"
|
20
|
+
app = ::QueueBus::Application.new(dispatcher.app_key)
|
21
|
+
app.subscribe(subscriptions, logging)
|
22
|
+
log ' ...done'
|
21
23
|
end
|
22
24
|
count
|
23
25
|
end
|
24
|
-
|
26
|
+
|
25
27
|
def unsubscribe!
|
26
28
|
count = 0
|
27
29
|
::QueueBus.dispatchers.each do |dispatcher|
|
@@ -29,10 +31,10 @@ module QueueBus
|
|
29
31
|
app = ::QueueBus::Application.new(dispatcher.app_key)
|
30
32
|
app.unsubscribe
|
31
33
|
count += 1
|
32
|
-
log
|
34
|
+
log ' ...done'
|
33
35
|
end
|
34
36
|
end
|
35
|
-
|
37
|
+
|
36
38
|
def queue_names
|
37
39
|
# let's not talk to redis in here. Seems to screw things up
|
38
40
|
queues = []
|
@@ -41,12 +43,12 @@ module QueueBus
|
|
41
43
|
queues << sub.queue_name
|
42
44
|
end
|
43
45
|
end
|
44
|
-
|
46
|
+
|
45
47
|
queues.uniq
|
46
48
|
end
|
47
|
-
|
49
|
+
|
48
50
|
def log(message)
|
49
51
|
puts(message) if logging
|
50
52
|
end
|
51
53
|
end
|
52
|
-
end
|
54
|
+
end
|
data/lib/queue_bus/tasks.rb
CHANGED
@@ -1,36 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# require 'queue_bus/tasks'
|
2
|
-
#
|
4
|
+
# A useful set of rake tasks for managing your bus
|
3
5
|
|
6
|
+
# rubocop:disable Metrics/BlockLength
|
4
7
|
namespace :queuebus do
|
5
|
-
|
6
|
-
|
7
|
-
task :subscribe => [ :preload ] do
|
8
|
+
desc 'Subscribes this application to QueueBus events'
|
9
|
+
task subscribe: [:preload] do
|
8
10
|
manager = ::QueueBus::TaskManager.new(true)
|
9
11
|
count = manager.subscribe!
|
10
|
-
raise
|
12
|
+
raise 'No subscriptions created' if count == 0
|
11
13
|
end
|
12
14
|
|
13
|
-
desc
|
14
|
-
task :
|
15
|
+
desc 'Unsubscribes this application from QueueBus events'
|
16
|
+
task unsubscribe: [:preload] do
|
15
17
|
manager = ::QueueBus::TaskManager.new(true)
|
16
18
|
count = manager.unsubscribe!
|
17
|
-
puts
|
19
|
+
puts 'No subscriptions unsubscribed' if count == 0
|
18
20
|
end
|
19
21
|
|
20
|
-
desc
|
21
|
-
task :
|
22
|
+
desc 'List QueueBus queues that need worked'
|
23
|
+
task queues: [:preload] do
|
22
24
|
manager = ::QueueBus::TaskManager.new(false)
|
23
25
|
queues = manager.queue_names + ['bus_incoming']
|
24
|
-
puts queues.join(
|
26
|
+
puts queues.join(', ')
|
25
27
|
end
|
26
28
|
|
27
|
-
desc
|
28
|
-
task :
|
29
|
+
desc 'list time based subscriptions'
|
30
|
+
task list_scheduled: [:preload] do
|
29
31
|
scheduled_list = QueueBus::Application.all.flat_map do |app|
|
30
32
|
app.send(:subscriptions).all
|
31
|
-
|
33
|
+
.select { |s| s.matcher.filters['bus_event_type'] == 'heartbeat_minutes' }
|
34
|
+
end
|
35
|
+
scheduled_text_list = scheduled_list.collect do |e|
|
36
|
+
[e.key, e.matcher.filters['hour'] || '*', e.matcher.filters['minute'] || '*']
|
32
37
|
end
|
33
|
-
scheduled_text_list = scheduled_list.collect { |e| [ e.key, e.matcher.filters['hour'] || "*", e.matcher.filters['minute'] || "*" ] }
|
34
38
|
puts 'key, hour, minute'
|
35
39
|
puts scheduled_text_list.sort_by { |(_, hour, minute)| [hour.to_i, minute.to_i] }.map(&:to_csv)
|
36
40
|
end
|
@@ -41,3 +45,4 @@ namespace :queuebus do
|
|
41
45
|
require 'queue-bus'
|
42
46
|
end
|
43
47
|
end
|
48
|
+
# rubocop:enable Metrics/BlockLength
|
data/lib/queue_bus/util.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'multi_json'
|
2
4
|
|
3
5
|
module QueueBus
|
@@ -30,14 +32,14 @@ module QueueBus
|
|
30
32
|
raise DecodeException, e.message, e.backtrace
|
31
33
|
end
|
32
34
|
end
|
33
|
-
|
35
|
+
|
34
36
|
def underscore(camel_cased_word)
|
35
37
|
word = camel_cased_word.to_s.dup
|
36
38
|
word.gsub!('::', '/')
|
37
39
|
# word.gsub!(/(?:([A-Za-z\d])|^)(#{inflections.acronym_regex})(?=\b|[^a-z])/) { "#{$1}#{$1 && '_'}#{$2.downcase}" }
|
38
|
-
word.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
|
39
|
-
word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
|
40
|
-
word.tr!(
|
40
|
+
word.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
|
41
|
+
word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
|
42
|
+
word.tr!('-', '_')
|
41
43
|
word.downcase!
|
42
44
|
word
|
43
45
|
end
|
@@ -53,11 +55,11 @@ module QueueBus
|
|
53
55
|
# string = string.sub(/^[a-z\d]*/) { inflections.acronyms[$&] || $&.capitalize }
|
54
56
|
string = string.sub(/^[a-z\d]*/) { $&.capitalize }
|
55
57
|
# string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{inflections.acronyms[$2] || $2.capitalize}" }
|
56
|
-
string.gsub!(
|
57
|
-
string.gsub!(
|
58
|
+
string.gsub!(%r{(?:_|(/))([a-z\d]*)}i) { "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}" }
|
59
|
+
string.gsub!(%r{/}, '::')
|
58
60
|
string
|
59
61
|
end
|
60
|
-
|
62
|
+
|
61
63
|
def constantize(camel_cased_word)
|
62
64
|
names = camel_cased_word.split('::')
|
63
65
|
names.shift if names.empty? || names.first.empty?
|
@@ -75,6 +77,7 @@ module QueueBus
|
|
75
77
|
constant = constant.ancestors.inject do |const, ancestor|
|
76
78
|
break const if ancestor == Object
|
77
79
|
break ancestor if ancestor.const_defined?(name, false)
|
80
|
+
|
78
81
|
const
|
79
82
|
end
|
80
83
|
|
@@ -84,4 +87,4 @@ module QueueBus
|
|
84
87
|
end
|
85
88
|
end
|
86
89
|
end
|
87
|
-
end
|
90
|
+
end
|
data/lib/queue_bus/version.rb
CHANGED
data/lib/queue_bus/worker.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
4
|
class Worker
|
3
|
-
|
4
5
|
def self.perform(json)
|
5
6
|
klass = nil
|
6
7
|
attributes = ::QueueBus::Util.decode(json)
|
7
8
|
begin
|
8
|
-
class_name = attributes[
|
9
|
+
class_name = attributes['bus_class_proxy']
|
9
10
|
klass = ::QueueBus::Util.constantize(class_name)
|
10
11
|
rescue NameError
|
11
12
|
# not there anymore
|
data/queue-bus.gemspec
CHANGED
@@ -1,33 +1,34 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$:.push File.expand_path('lib', __dir__)
|
4
|
+
require 'queue_bus/version'
|
4
5
|
|
5
6
|
Gem::Specification.new do |s|
|
6
|
-
s.name =
|
7
|
+
s.name = 'queue-bus'
|
7
8
|
s.version = QueueBus::VERSION
|
8
|
-
s.authors = [
|
9
|
-
s.email = [
|
10
|
-
s.homepage =
|
11
|
-
s.summary =
|
12
|
-
s.description =
|
9
|
+
s.authors = ['Brian Leonard']
|
10
|
+
s.email = ['brian@bleonard.com']
|
11
|
+
s.homepage = ''
|
12
|
+
s.summary = 'A simple event bus on top of background queues'
|
13
|
+
s.description = 'A simple event bus on top of common background queues. Publish and subscribe to events as they occur using what you already have.'
|
13
14
|
|
14
|
-
s.rubyforge_project =
|
15
|
+
s.rubyforge_project = 'queue-bus'
|
15
16
|
|
16
17
|
s.files = `git ls-files`.split("\n")
|
17
18
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
-
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
-
s.require_paths = [
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
20
|
+
s.require_paths = ['lib']
|
20
21
|
|
21
|
-
s.add_dependency(
|
22
|
-
s.add_dependency(
|
22
|
+
s.add_dependency('multi_json')
|
23
|
+
s.add_dependency('redis')
|
23
24
|
|
24
25
|
# if using resque
|
25
26
|
# s.add_development_dependency('resque', ['>= 1.10.0', '< 2.0'])
|
26
27
|
# s.add_development_dependency('resque-scheduler', '>= 2.0.1')
|
27
28
|
# s.add_development_dependency('resque-retry')
|
28
29
|
|
29
|
-
s.add_development_dependency(
|
30
|
-
s.add_development_dependency(
|
31
|
-
s.add_development_dependency(
|
32
|
-
s.add_development_dependency(
|
30
|
+
s.add_development_dependency('json_pure')
|
31
|
+
s.add_development_dependency('rspec')
|
32
|
+
s.add_development_dependency('rubocop')
|
33
|
+
s.add_development_dependency('timecop')
|
33
34
|
end
|