queue-bus 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/CHANGELOG.md +9 -0
- data/Gemfile +4 -2
- data/Rakefile +2 -0
- data/lib/queue-bus.rb +15 -12
- data/lib/queue_bus/adapters/base.rb +4 -2
- data/lib/queue_bus/adapters/data.rb +12 -11
- data/lib/queue_bus/application.rb +13 -15
- data/lib/queue_bus/dispatch.rb +14 -12
- data/lib/queue_bus/dispatchers.rb +12 -5
- data/lib/queue_bus/driver.rb +15 -10
- data/lib/queue_bus/heartbeat.rb +32 -30
- data/lib/queue_bus/local.rb +9 -9
- data/lib/queue_bus/matcher.rb +36 -27
- data/lib/queue_bus/publisher.rb +7 -5
- data/lib/queue_bus/publishing.rb +31 -24
- data/lib/queue_bus/rider.rb +26 -22
- data/lib/queue_bus/subscriber.rb +20 -14
- data/lib/queue_bus/subscription.rb +25 -15
- data/lib/queue_bus/subscription_list.rb +30 -12
- data/lib/queue_bus/task_manager.rb +18 -16
- data/lib/queue_bus/tasks.rb +20 -15
- data/lib/queue_bus/util.rb +11 -8
- data/lib/queue_bus/version.rb +3 -1
- data/lib/queue_bus/worker.rb +3 -2
- data/queue-bus.gemspec +19 -18
- data/spec/adapter/publish_at_spec.rb +28 -25
- data/spec/adapter/support.rb +7 -1
- data/spec/adapter_spec.rb +4 -2
- data/spec/application_spec.rb +97 -97
- data/spec/dispatch_spec.rb +48 -51
- data/spec/driver_spec.rb +60 -58
- data/spec/heartbeat_spec.rb +26 -24
- data/spec/integration_spec.rb +41 -40
- data/spec/matcher_spec.rb +104 -102
- data/spec/publish_spec.rb +46 -46
- data/spec/publisher_spec.rb +3 -1
- data/spec/rider_spec.rb +16 -14
- data/spec/spec_helper.rb +2 -2
- data/spec/subscriber_spec.rb +227 -227
- data/spec/subscription_list_spec.rb +31 -31
- data/spec/subscription_spec.rb +37 -36
- data/spec/worker_spec.rb +17 -15
- metadata +7 -6
data/lib/queue_bus/subscriber.rb
CHANGED
@@ -1,28 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A mixin to configure subscriptions on a particular class
|
2
5
|
module Subscriber
|
3
|
-
|
4
6
|
def self.included(base)
|
5
7
|
base.extend ClassMethods
|
6
8
|
end
|
7
9
|
|
10
|
+
# The class methods that will be added to the class it's included in. Use them to
|
11
|
+
# configure and subscribe.
|
8
12
|
module ClassMethods
|
9
|
-
|
10
13
|
def application(app_key)
|
11
14
|
@app_key = ::QueueBus::Application.normalize(app_key)
|
12
15
|
end
|
13
16
|
|
14
17
|
def app_key
|
15
18
|
return @app_key if @app_key
|
19
|
+
|
16
20
|
@app_key = ::QueueBus.default_app_key
|
17
21
|
return @app_key if @app_key
|
22
|
+
|
18
23
|
# module or class_name
|
19
|
-
val =
|
24
|
+
val = name.to_s.split('::').first
|
20
25
|
@app_key = ::QueueBus::Util.underscore(val)
|
21
26
|
end
|
22
27
|
|
23
28
|
def subscribe(method_name, matcher_hash = nil)
|
24
29
|
queue_name = nil
|
25
|
-
queue_name ||=
|
30
|
+
queue_name ||= instance_variable_get(:@queue) || (respond_to?(:queue) && queue)
|
26
31
|
queue_name ||= ::QueueBus.default_queue
|
27
32
|
queue_name ||= "#{app_key}_default"
|
28
33
|
subscribe_queue(queue_name, method_name, matcher_hash)
|
@@ -30,10 +35,11 @@ module QueueBus
|
|
30
35
|
|
31
36
|
def subscribe_queue(queue_name, method_name, matcher_hash = nil)
|
32
37
|
klass = self
|
33
|
-
matcher_hash ||= {
|
34
|
-
sub_key = "#{
|
38
|
+
matcher_hash ||= { 'bus_event_type' => method_name }
|
39
|
+
sub_key = "#{name}.#{method_name}"
|
35
40
|
dispatcher = ::QueueBus.dispatcher_by_key(app_key)
|
36
|
-
dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash,
|
41
|
+
dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash,
|
42
|
+
->(att) { klass.perform(att) })
|
37
43
|
end
|
38
44
|
|
39
45
|
def transform(method_name)
|
@@ -42,8 +48,8 @@ module QueueBus
|
|
42
48
|
|
43
49
|
def perform(attributes)
|
44
50
|
::QueueBus.with_global_attributes(attributes) do
|
45
|
-
sub_key = attributes[
|
46
|
-
meth_key = sub_key.split(
|
51
|
+
sub_key = attributes['bus_rider_sub_key']
|
52
|
+
meth_key = sub_key.split('.').last
|
47
53
|
queue_bus_execute(meth_key, attributes)
|
48
54
|
end
|
49
55
|
end
|
@@ -52,11 +58,11 @@ module QueueBus
|
|
52
58
|
args = attributes
|
53
59
|
args = send(@transform, attributes) if @transform
|
54
60
|
args = [args] unless args.is_a?(Array)
|
55
|
-
if
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
61
|
+
me = if respond_to?(:subscriber_with_attributes)
|
62
|
+
subscriber_with_attributes(attributes)
|
63
|
+
else
|
64
|
+
new
|
65
|
+
end
|
60
66
|
me.send(key, *args)
|
61
67
|
end
|
62
68
|
end
|
@@ -1,29 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A Subscription is the destination of an event.
|
5
|
+
#
|
6
|
+
# The subscription can be stored in redis but should only be executed on ruby processes that
|
7
|
+
# have the application loaded. In general, this is controlled by having the background workers
|
8
|
+
# listen to specific (and discrete) queues.
|
2
9
|
class Subscription
|
3
|
-
|
4
10
|
class << self
|
5
11
|
def register(queue, key, class_name, matcher, block)
|
6
12
|
Subscription.new(queue, key, class_name, matcher, block)
|
7
13
|
end
|
8
14
|
|
9
15
|
def from_redis(hash)
|
10
|
-
queue_name = hash[
|
11
|
-
key = hash[
|
12
|
-
class_name = hash[
|
13
|
-
matcher = hash[
|
14
|
-
return nil if key.
|
16
|
+
queue_name = hash['queue_name'].to_s
|
17
|
+
key = hash['key'].to_s
|
18
|
+
class_name = hash['class'].to_s
|
19
|
+
matcher = hash['matcher']
|
20
|
+
return nil if key.empty? || queue_name.empty?
|
21
|
+
|
15
22
|
Subscription.new(queue_name, key, class_name, matcher, nil)
|
16
23
|
end
|
17
24
|
|
18
25
|
def normalize(val)
|
19
|
-
val.to_s.gsub(/\W/,
|
26
|
+
val.to_s.gsub(/\W/, '_').downcase
|
20
27
|
end
|
21
28
|
end
|
22
29
|
|
23
30
|
attr_reader :matcher, :executor, :queue_name, :key, :class_name
|
24
|
-
attr_accessor :app_key
|
31
|
+
attr_accessor :app_key # dyanmically set on return from subscription_matches
|
25
32
|
|
26
|
-
def initialize(queue_name, key, class_name, filters, executor=nil)
|
33
|
+
def initialize(queue_name, key, class_name, filters, executor = nil)
|
27
34
|
@queue_name = self.class.normalize(queue_name)
|
28
35
|
@key = key.to_s
|
29
36
|
@class_name = class_name.to_s
|
@@ -31,8 +38,12 @@ module QueueBus
|
|
31
38
|
@executor = executor
|
32
39
|
end
|
33
40
|
|
41
|
+
# Executes the subscription. If this is run on a server/ruby process that did not subscribe
|
42
|
+
# it will error as there will not be a proc.
|
34
43
|
def execute!(attributes)
|
35
|
-
|
44
|
+
if attributes.respond_to?(:with_indifferent_access)
|
45
|
+
attributes = attributes.with_indifferent_access
|
46
|
+
end
|
36
47
|
::QueueBus.with_global_attributes(attributes) do
|
37
48
|
executor.call(attributes)
|
38
49
|
end
|
@@ -44,12 +55,11 @@ module QueueBus
|
|
44
55
|
|
45
56
|
def to_redis
|
46
57
|
out = {}
|
47
|
-
out[
|
48
|
-
out[
|
49
|
-
out[
|
50
|
-
out[
|
58
|
+
out['queue_name'] = queue_name
|
59
|
+
out['key'] = key
|
60
|
+
out['class'] = class_name
|
61
|
+
out['matcher'] = matcher.to_redis
|
51
62
|
out
|
52
63
|
end
|
53
|
-
|
54
64
|
end
|
55
65
|
end
|
@@ -1,11 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# Manages a set of subscriptions.
|
5
|
+
#
|
6
|
+
# The subscriptions are stored in redis but not by this class. Instead this class uses two
|
7
|
+
# functions `to_redis` and `from_redis` to facilitate serialization without accessing redis
|
8
|
+
# directly.
|
9
|
+
#
|
10
|
+
# To create a new SubscriptionList, use the static function `from_redis` and pass
|
11
|
+
# it a hash that came from redis.
|
12
|
+
#
|
13
|
+
# To get a value fro redis, take your loaded SubscriptionList and call `to_redis` on it. The
|
14
|
+
# returned value can be used to store in redis.
|
2
15
|
class SubscriptionList
|
3
|
-
|
4
16
|
class << self
|
5
17
|
def from_redis(redis_hash)
|
6
18
|
out = SubscriptionList.new
|
7
|
-
|
8
|
-
redis_hash.each do |
|
19
|
+
|
20
|
+
redis_hash.each do |_key, value|
|
9
21
|
sub = Subscription.from_redis(value)
|
10
22
|
out.add(sub) if sub
|
11
23
|
end
|
@@ -21,29 +33,35 @@ module QueueBus
|
|
21
33
|
end
|
22
34
|
out
|
23
35
|
end
|
24
|
-
|
36
|
+
|
25
37
|
def initialize
|
26
38
|
@subscriptions = {}
|
27
39
|
end
|
28
|
-
|
40
|
+
|
29
41
|
def add(sub)
|
30
|
-
|
31
|
-
|
42
|
+
if @subscriptions.key?(sub.key)
|
43
|
+
raise "Duplicate key: #{sub.key} already exists " \
|
44
|
+
"in the #{sub.queue_name} queue!"
|
45
|
+
end
|
32
46
|
@subscriptions[sub.key] = sub
|
33
47
|
end
|
34
|
-
|
48
|
+
|
35
49
|
def size
|
36
50
|
@subscriptions.size
|
37
51
|
end
|
38
|
-
|
52
|
+
|
53
|
+
def empty?
|
54
|
+
size.zero?
|
55
|
+
end
|
56
|
+
|
39
57
|
def key(key)
|
40
58
|
@subscriptions[key.to_s]
|
41
59
|
end
|
42
|
-
|
60
|
+
|
43
61
|
def all
|
44
62
|
@subscriptions.values
|
45
63
|
end
|
46
|
-
|
64
|
+
|
47
65
|
def matches(attributes)
|
48
66
|
out = []
|
49
67
|
all.each do |sub|
|
@@ -52,4 +70,4 @@ module QueueBus
|
|
52
70
|
out
|
53
71
|
end
|
54
72
|
end
|
55
|
-
end
|
73
|
+
end
|
@@ -1,27 +1,29 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A helper class for executing Rake tasks.
|
2
5
|
class TaskManager
|
3
|
-
|
4
6
|
attr_reader :logging
|
5
|
-
|
7
|
+
|
6
8
|
def initialize(logging)
|
7
9
|
@logging = logging
|
8
10
|
end
|
9
|
-
|
11
|
+
|
10
12
|
def subscribe!
|
11
13
|
count = 0
|
12
14
|
::QueueBus.dispatchers.each do |dispatcher|
|
13
15
|
subscriptions = dispatcher.subscriptions
|
14
|
-
if subscriptions.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
next if subscriptions.empty?
|
17
|
+
|
18
|
+
count += subscriptions.size
|
19
|
+
log "Subscribing #{dispatcher.app_key} to #{subscriptions.size} subscriptions"
|
20
|
+
app = ::QueueBus::Application.new(dispatcher.app_key)
|
21
|
+
app.subscribe(subscriptions, logging)
|
22
|
+
log ' ...done'
|
21
23
|
end
|
22
24
|
count
|
23
25
|
end
|
24
|
-
|
26
|
+
|
25
27
|
def unsubscribe!
|
26
28
|
count = 0
|
27
29
|
::QueueBus.dispatchers.each do |dispatcher|
|
@@ -29,10 +31,10 @@ module QueueBus
|
|
29
31
|
app = ::QueueBus::Application.new(dispatcher.app_key)
|
30
32
|
app.unsubscribe
|
31
33
|
count += 1
|
32
|
-
log
|
34
|
+
log ' ...done'
|
33
35
|
end
|
34
36
|
end
|
35
|
-
|
37
|
+
|
36
38
|
def queue_names
|
37
39
|
# let's not talk to redis in here. Seems to screw things up
|
38
40
|
queues = []
|
@@ -41,12 +43,12 @@ module QueueBus
|
|
41
43
|
queues << sub.queue_name
|
42
44
|
end
|
43
45
|
end
|
44
|
-
|
46
|
+
|
45
47
|
queues.uniq
|
46
48
|
end
|
47
|
-
|
49
|
+
|
48
50
|
def log(message)
|
49
51
|
puts(message) if logging
|
50
52
|
end
|
51
53
|
end
|
52
|
-
end
|
54
|
+
end
|
data/lib/queue_bus/tasks.rb
CHANGED
@@ -1,36 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# require 'queue_bus/tasks'
|
2
|
-
#
|
4
|
+
# A useful set of rake tasks for managing your bus
|
3
5
|
|
6
|
+
# rubocop:disable Metrics/BlockLength
|
4
7
|
namespace :queuebus do
|
5
|
-
|
6
|
-
|
7
|
-
task :subscribe => [ :preload ] do
|
8
|
+
desc 'Subscribes this application to QueueBus events'
|
9
|
+
task subscribe: [:preload] do
|
8
10
|
manager = ::QueueBus::TaskManager.new(true)
|
9
11
|
count = manager.subscribe!
|
10
|
-
raise
|
12
|
+
raise 'No subscriptions created' if count == 0
|
11
13
|
end
|
12
14
|
|
13
|
-
desc
|
14
|
-
task :
|
15
|
+
desc 'Unsubscribes this application from QueueBus events'
|
16
|
+
task unsubscribe: [:preload] do
|
15
17
|
manager = ::QueueBus::TaskManager.new(true)
|
16
18
|
count = manager.unsubscribe!
|
17
|
-
puts
|
19
|
+
puts 'No subscriptions unsubscribed' if count == 0
|
18
20
|
end
|
19
21
|
|
20
|
-
desc
|
21
|
-
task :
|
22
|
+
desc 'List QueueBus queues that need worked'
|
23
|
+
task queues: [:preload] do
|
22
24
|
manager = ::QueueBus::TaskManager.new(false)
|
23
25
|
queues = manager.queue_names + ['bus_incoming']
|
24
|
-
puts queues.join(
|
26
|
+
puts queues.join(', ')
|
25
27
|
end
|
26
28
|
|
27
|
-
desc
|
28
|
-
task :
|
29
|
+
desc 'list time based subscriptions'
|
30
|
+
task list_scheduled: [:preload] do
|
29
31
|
scheduled_list = QueueBus::Application.all.flat_map do |app|
|
30
32
|
app.send(:subscriptions).all
|
31
|
-
|
33
|
+
.select { |s| s.matcher.filters['bus_event_type'] == 'heartbeat_minutes' }
|
34
|
+
end
|
35
|
+
scheduled_text_list = scheduled_list.collect do |e|
|
36
|
+
[e.key, e.matcher.filters['hour'] || '*', e.matcher.filters['minute'] || '*']
|
32
37
|
end
|
33
|
-
scheduled_text_list = scheduled_list.collect { |e| [ e.key, e.matcher.filters['hour'] || "*", e.matcher.filters['minute'] || "*" ] }
|
34
38
|
puts 'key, hour, minute'
|
35
39
|
puts scheduled_text_list.sort_by { |(_, hour, minute)| [hour.to_i, minute.to_i] }.map(&:to_csv)
|
36
40
|
end
|
@@ -41,3 +45,4 @@ namespace :queuebus do
|
|
41
45
|
require 'queue-bus'
|
42
46
|
end
|
43
47
|
end
|
48
|
+
# rubocop:enable Metrics/BlockLength
|
data/lib/queue_bus/util.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'multi_json'
|
2
4
|
|
3
5
|
module QueueBus
|
@@ -30,14 +32,14 @@ module QueueBus
|
|
30
32
|
raise DecodeException, e.message, e.backtrace
|
31
33
|
end
|
32
34
|
end
|
33
|
-
|
35
|
+
|
34
36
|
def underscore(camel_cased_word)
|
35
37
|
word = camel_cased_word.to_s.dup
|
36
38
|
word.gsub!('::', '/')
|
37
39
|
# word.gsub!(/(?:([A-Za-z\d])|^)(#{inflections.acronym_regex})(?=\b|[^a-z])/) { "#{$1}#{$1 && '_'}#{$2.downcase}" }
|
38
|
-
word.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
|
39
|
-
word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
|
40
|
-
word.tr!(
|
40
|
+
word.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
|
41
|
+
word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
|
42
|
+
word.tr!('-', '_')
|
41
43
|
word.downcase!
|
42
44
|
word
|
43
45
|
end
|
@@ -53,11 +55,11 @@ module QueueBus
|
|
53
55
|
# string = string.sub(/^[a-z\d]*/) { inflections.acronyms[$&] || $&.capitalize }
|
54
56
|
string = string.sub(/^[a-z\d]*/) { $&.capitalize }
|
55
57
|
# string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{inflections.acronyms[$2] || $2.capitalize}" }
|
56
|
-
string.gsub!(
|
57
|
-
string.gsub!(
|
58
|
+
string.gsub!(%r{(?:_|(/))([a-z\d]*)}i) { "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}" }
|
59
|
+
string.gsub!(%r{/}, '::')
|
58
60
|
string
|
59
61
|
end
|
60
|
-
|
62
|
+
|
61
63
|
def constantize(camel_cased_word)
|
62
64
|
names = camel_cased_word.split('::')
|
63
65
|
names.shift if names.empty? || names.first.empty?
|
@@ -75,6 +77,7 @@ module QueueBus
|
|
75
77
|
constant = constant.ancestors.inject do |const, ancestor|
|
76
78
|
break const if ancestor == Object
|
77
79
|
break ancestor if ancestor.const_defined?(name, false)
|
80
|
+
|
78
81
|
const
|
79
82
|
end
|
80
83
|
|
@@ -84,4 +87,4 @@ module QueueBus
|
|
84
87
|
end
|
85
88
|
end
|
86
89
|
end
|
87
|
-
end
|
90
|
+
end
|
data/lib/queue_bus/version.rb
CHANGED
data/lib/queue_bus/worker.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
4
|
class Worker
|
3
|
-
|
4
5
|
def self.perform(json)
|
5
6
|
klass = nil
|
6
7
|
attributes = ::QueueBus::Util.decode(json)
|
7
8
|
begin
|
8
|
-
class_name = attributes[
|
9
|
+
class_name = attributes['bus_class_proxy']
|
9
10
|
klass = ::QueueBus::Util.constantize(class_name)
|
10
11
|
rescue NameError
|
11
12
|
# not there anymore
|
data/queue-bus.gemspec
CHANGED
@@ -1,33 +1,34 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
$:.push File.expand_path('lib', __dir__)
|
4
|
+
require 'queue_bus/version'
|
4
5
|
|
5
6
|
Gem::Specification.new do |s|
|
6
|
-
s.name =
|
7
|
+
s.name = 'queue-bus'
|
7
8
|
s.version = QueueBus::VERSION
|
8
|
-
s.authors = [
|
9
|
-
s.email = [
|
10
|
-
s.homepage =
|
11
|
-
s.summary =
|
12
|
-
s.description =
|
9
|
+
s.authors = ['Brian Leonard']
|
10
|
+
s.email = ['brian@bleonard.com']
|
11
|
+
s.homepage = ''
|
12
|
+
s.summary = 'A simple event bus on top of background queues'
|
13
|
+
s.description = 'A simple event bus on top of common background queues. Publish and subscribe to events as they occur using what you already have.'
|
13
14
|
|
14
|
-
s.rubyforge_project =
|
15
|
+
s.rubyforge_project = 'queue-bus'
|
15
16
|
|
16
17
|
s.files = `git ls-files`.split("\n")
|
17
18
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
-
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
-
s.require_paths = [
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
|
20
|
+
s.require_paths = ['lib']
|
20
21
|
|
21
|
-
s.add_dependency(
|
22
|
-
s.add_dependency(
|
22
|
+
s.add_dependency('multi_json')
|
23
|
+
s.add_dependency('redis')
|
23
24
|
|
24
25
|
# if using resque
|
25
26
|
# s.add_development_dependency('resque', ['>= 1.10.0', '< 2.0'])
|
26
27
|
# s.add_development_dependency('resque-scheduler', '>= 2.0.1')
|
27
28
|
# s.add_development_dependency('resque-retry')
|
28
29
|
|
29
|
-
s.add_development_dependency(
|
30
|
-
s.add_development_dependency(
|
31
|
-
s.add_development_dependency(
|
32
|
-
s.add_development_dependency(
|
30
|
+
s.add_development_dependency('json_pure')
|
31
|
+
s.add_development_dependency('rspec')
|
32
|
+
s.add_development_dependency('rubocop')
|
33
|
+
s.add_development_dependency('timecop')
|
33
34
|
end
|