queue-bus 0.9.0 → 0.9.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +21 -0
  3. data/CHANGELOG.md +9 -0
  4. data/Gemfile +4 -2
  5. data/Rakefile +2 -0
  6. data/lib/queue-bus.rb +15 -12
  7. data/lib/queue_bus/adapters/base.rb +4 -2
  8. data/lib/queue_bus/adapters/data.rb +12 -11
  9. data/lib/queue_bus/application.rb +13 -15
  10. data/lib/queue_bus/dispatch.rb +14 -12
  11. data/lib/queue_bus/dispatchers.rb +12 -5
  12. data/lib/queue_bus/driver.rb +15 -10
  13. data/lib/queue_bus/heartbeat.rb +32 -30
  14. data/lib/queue_bus/local.rb +9 -9
  15. data/lib/queue_bus/matcher.rb +36 -27
  16. data/lib/queue_bus/publisher.rb +7 -5
  17. data/lib/queue_bus/publishing.rb +31 -24
  18. data/lib/queue_bus/rider.rb +26 -22
  19. data/lib/queue_bus/subscriber.rb +20 -14
  20. data/lib/queue_bus/subscription.rb +25 -15
  21. data/lib/queue_bus/subscription_list.rb +30 -12
  22. data/lib/queue_bus/task_manager.rb +18 -16
  23. data/lib/queue_bus/tasks.rb +20 -15
  24. data/lib/queue_bus/util.rb +11 -8
  25. data/lib/queue_bus/version.rb +3 -1
  26. data/lib/queue_bus/worker.rb +3 -2
  27. data/queue-bus.gemspec +19 -18
  28. data/spec/adapter/publish_at_spec.rb +28 -25
  29. data/spec/adapter/support.rb +7 -1
  30. data/spec/adapter_spec.rb +4 -2
  31. data/spec/application_spec.rb +97 -97
  32. data/spec/dispatch_spec.rb +48 -51
  33. data/spec/driver_spec.rb +60 -58
  34. data/spec/heartbeat_spec.rb +26 -24
  35. data/spec/integration_spec.rb +41 -40
  36. data/spec/matcher_spec.rb +104 -102
  37. data/spec/publish_spec.rb +46 -46
  38. data/spec/publisher_spec.rb +3 -1
  39. data/spec/rider_spec.rb +16 -14
  40. data/spec/spec_helper.rb +2 -2
  41. data/spec/subscriber_spec.rb +227 -227
  42. data/spec/subscription_list_spec.rb +31 -31
  43. data/spec/subscription_spec.rb +37 -36
  44. data/spec/worker_spec.rb +17 -15
  45. metadata +7 -6
@@ -1,28 +1,33 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # A mixin to configure subscriptions on a particular class
2
5
  module Subscriber
3
-
4
6
  def self.included(base)
5
7
  base.extend ClassMethods
6
8
  end
7
9
 
10
+ # The class methods that will be added to the class it's included in. Use them to
11
+ # configure and subscribe.
8
12
  module ClassMethods
9
-
10
13
  def application(app_key)
11
14
  @app_key = ::QueueBus::Application.normalize(app_key)
12
15
  end
13
16
 
14
17
  def app_key
15
18
  return @app_key if @app_key
19
+
16
20
  @app_key = ::QueueBus.default_app_key
17
21
  return @app_key if @app_key
22
+
18
23
  # module or class_name
19
- val = self.name.to_s.split("::").first
24
+ val = name.to_s.split('::').first
20
25
  @app_key = ::QueueBus::Util.underscore(val)
21
26
  end
22
27
 
23
28
  def subscribe(method_name, matcher_hash = nil)
24
29
  queue_name = nil
25
- queue_name ||= self.instance_variable_get(:@queue) || (self.respond_to?(:queue) && self.queue)
30
+ queue_name ||= instance_variable_get(:@queue) || (respond_to?(:queue) && queue)
26
31
  queue_name ||= ::QueueBus.default_queue
27
32
  queue_name ||= "#{app_key}_default"
28
33
  subscribe_queue(queue_name, method_name, matcher_hash)
@@ -30,10 +35,11 @@ module QueueBus
30
35
 
31
36
  def subscribe_queue(queue_name, method_name, matcher_hash = nil)
32
37
  klass = self
33
- matcher_hash ||= {"bus_event_type" => method_name}
34
- sub_key = "#{self.name}.#{method_name}"
38
+ matcher_hash ||= { 'bus_event_type' => method_name }
39
+ sub_key = "#{name}.#{method_name}"
35
40
  dispatcher = ::QueueBus.dispatcher_by_key(app_key)
36
- dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash, lambda{ |att| klass.perform(att) })
41
+ dispatcher.add_subscription(queue_name, sub_key, klass.name.to_s, matcher_hash,
42
+ ->(att) { klass.perform(att) })
37
43
  end
38
44
 
39
45
  def transform(method_name)
@@ -42,8 +48,8 @@ module QueueBus
42
48
 
43
49
  def perform(attributes)
44
50
  ::QueueBus.with_global_attributes(attributes) do
45
- sub_key = attributes["bus_rider_sub_key"]
46
- meth_key = sub_key.split(".").last
51
+ sub_key = attributes['bus_rider_sub_key']
52
+ meth_key = sub_key.split('.').last
47
53
  queue_bus_execute(meth_key, attributes)
48
54
  end
49
55
  end
@@ -52,11 +58,11 @@ module QueueBus
52
58
  args = attributes
53
59
  args = send(@transform, attributes) if @transform
54
60
  args = [args] unless args.is_a?(Array)
55
- if self.respond_to?(:subscriber_with_attributes)
56
- me = self.subscriber_with_attributes(attributes)
57
- else
58
- me = self.new
59
- end
61
+ me = if respond_to?(:subscriber_with_attributes)
62
+ subscriber_with_attributes(attributes)
63
+ else
64
+ new
65
+ end
60
66
  me.send(key, *args)
61
67
  end
62
68
  end
@@ -1,29 +1,36 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # A Subscription is the destination of an event.
5
+ #
6
+ # The subscription can be stored in redis but should only be executed on ruby processes that
7
+ # have the application loaded. In general, this is controlled by having the background workers
8
+ # listen to specific (and discrete) queues.
2
9
  class Subscription
3
-
4
10
  class << self
5
11
  def register(queue, key, class_name, matcher, block)
6
12
  Subscription.new(queue, key, class_name, matcher, block)
7
13
  end
8
14
 
9
15
  def from_redis(hash)
10
- queue_name = hash["queue_name"].to_s
11
- key = hash["key"].to_s
12
- class_name = hash["class"].to_s
13
- matcher = hash["matcher"]
14
- return nil if key.length == 0 || queue_name.length == 0
16
+ queue_name = hash['queue_name'].to_s
17
+ key = hash['key'].to_s
18
+ class_name = hash['class'].to_s
19
+ matcher = hash['matcher']
20
+ return nil if key.empty? || queue_name.empty?
21
+
15
22
  Subscription.new(queue_name, key, class_name, matcher, nil)
16
23
  end
17
24
 
18
25
  def normalize(val)
19
- val.to_s.gsub(/\W/, "_").downcase
26
+ val.to_s.gsub(/\W/, '_').downcase
20
27
  end
21
28
  end
22
29
 
23
30
  attr_reader :matcher, :executor, :queue_name, :key, :class_name
24
- attr_accessor :app_key # dyanmically set on return from subscription_matches
31
+ attr_accessor :app_key # dyanmically set on return from subscription_matches
25
32
 
26
- def initialize(queue_name, key, class_name, filters, executor=nil)
33
+ def initialize(queue_name, key, class_name, filters, executor = nil)
27
34
  @queue_name = self.class.normalize(queue_name)
28
35
  @key = key.to_s
29
36
  @class_name = class_name.to_s
@@ -31,8 +38,12 @@ module QueueBus
31
38
  @executor = executor
32
39
  end
33
40
 
41
+ # Executes the subscription. If this is run on a server/ruby process that did not subscribe
42
+ # it will error as there will not be a proc.
34
43
  def execute!(attributes)
35
- attributes = attributes.with_indifferent_access if attributes.respond_to?(:with_indifferent_access)
44
+ if attributes.respond_to?(:with_indifferent_access)
45
+ attributes = attributes.with_indifferent_access
46
+ end
36
47
  ::QueueBus.with_global_attributes(attributes) do
37
48
  executor.call(attributes)
38
49
  end
@@ -44,12 +55,11 @@ module QueueBus
44
55
 
45
56
  def to_redis
46
57
  out = {}
47
- out["queue_name"] = queue_name
48
- out["key"] = key
49
- out["class"] = class_name
50
- out["matcher"] = matcher.to_redis
58
+ out['queue_name'] = queue_name
59
+ out['key'] = key
60
+ out['class'] = class_name
61
+ out['matcher'] = matcher.to_redis
51
62
  out
52
63
  end
53
-
54
64
  end
55
65
  end
@@ -1,11 +1,23 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # Manages a set of subscriptions.
5
+ #
6
+ # The subscriptions are stored in redis but not by this class. Instead this class uses two
7
+ # functions `to_redis` and `from_redis` to facilitate serialization without accessing redis
8
+ # directly.
9
+ #
10
+ # To create a new SubscriptionList, use the static function `from_redis` and pass
11
+ # it a hash that came from redis.
12
+ #
13
+ # To get a value fro redis, take your loaded SubscriptionList and call `to_redis` on it. The
14
+ # returned value can be used to store in redis.
2
15
  class SubscriptionList
3
-
4
16
  class << self
5
17
  def from_redis(redis_hash)
6
18
  out = SubscriptionList.new
7
-
8
- redis_hash.each do |key, value|
19
+
20
+ redis_hash.each do |_key, value|
9
21
  sub = Subscription.from_redis(value)
10
22
  out.add(sub) if sub
11
23
  end
@@ -21,29 +33,35 @@ module QueueBus
21
33
  end
22
34
  out
23
35
  end
24
-
36
+
25
37
  def initialize
26
38
  @subscriptions = {}
27
39
  end
28
-
40
+
29
41
  def add(sub)
30
- raise "Duplicate key: #{sub.key} already exists " \
31
- "in the #{sub.queue_name} queue!" if @subscriptions.key?(sub.key)
42
+ if @subscriptions.key?(sub.key)
43
+ raise "Duplicate key: #{sub.key} already exists " \
44
+ "in the #{sub.queue_name} queue!"
45
+ end
32
46
  @subscriptions[sub.key] = sub
33
47
  end
34
-
48
+
35
49
  def size
36
50
  @subscriptions.size
37
51
  end
38
-
52
+
53
+ def empty?
54
+ size.zero?
55
+ end
56
+
39
57
  def key(key)
40
58
  @subscriptions[key.to_s]
41
59
  end
42
-
60
+
43
61
  def all
44
62
  @subscriptions.values
45
63
  end
46
-
64
+
47
65
  def matches(attributes)
48
66
  out = []
49
67
  all.each do |sub|
@@ -52,4 +70,4 @@ module QueueBus
52
70
  out
53
71
  end
54
72
  end
55
- end
73
+ end
@@ -1,27 +1,29 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # A helper class for executing Rake tasks.
2
5
  class TaskManager
3
-
4
6
  attr_reader :logging
5
-
7
+
6
8
  def initialize(logging)
7
9
  @logging = logging
8
10
  end
9
-
11
+
10
12
  def subscribe!
11
13
  count = 0
12
14
  ::QueueBus.dispatchers.each do |dispatcher|
13
15
  subscriptions = dispatcher.subscriptions
14
- if subscriptions.size > 0
15
- count += subscriptions.size
16
- log "Subscribing #{dispatcher.app_key} to #{subscriptions.size} subscriptions"
17
- app = ::QueueBus::Application.new(dispatcher.app_key)
18
- app.subscribe(subscriptions, logging)
19
- log " ...done"
20
- end
16
+ next if subscriptions.empty?
17
+
18
+ count += subscriptions.size
19
+ log "Subscribing #{dispatcher.app_key} to #{subscriptions.size} subscriptions"
20
+ app = ::QueueBus::Application.new(dispatcher.app_key)
21
+ app.subscribe(subscriptions, logging)
22
+ log ' ...done'
21
23
  end
22
24
  count
23
25
  end
24
-
26
+
25
27
  def unsubscribe!
26
28
  count = 0
27
29
  ::QueueBus.dispatchers.each do |dispatcher|
@@ -29,10 +31,10 @@ module QueueBus
29
31
  app = ::QueueBus::Application.new(dispatcher.app_key)
30
32
  app.unsubscribe
31
33
  count += 1
32
- log " ...done"
34
+ log ' ...done'
33
35
  end
34
36
  end
35
-
37
+
36
38
  def queue_names
37
39
  # let's not talk to redis in here. Seems to screw things up
38
40
  queues = []
@@ -41,12 +43,12 @@ module QueueBus
41
43
  queues << sub.queue_name
42
44
  end
43
45
  end
44
-
46
+
45
47
  queues.uniq
46
48
  end
47
-
49
+
48
50
  def log(message)
49
51
  puts(message) if logging
50
52
  end
51
53
  end
52
- end
54
+ end
@@ -1,36 +1,40 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # require 'queue_bus/tasks'
2
- # will give you these tasks
4
+ # A useful set of rake tasks for managing your bus
3
5
 
6
+ # rubocop:disable Metrics/BlockLength
4
7
  namespace :queuebus do
5
-
6
- desc "Subscribes this application to QueueBus events"
7
- task :subscribe => [ :preload ] do
8
+ desc 'Subscribes this application to QueueBus events'
9
+ task subscribe: [:preload] do
8
10
  manager = ::QueueBus::TaskManager.new(true)
9
11
  count = manager.subscribe!
10
- raise "No subscriptions created" if count == 0
12
+ raise 'No subscriptions created' if count == 0
11
13
  end
12
14
 
13
- desc "Unsubscribes this application from QueueBus events"
14
- task :unsubscribe => [ :preload ] do
15
+ desc 'Unsubscribes this application from QueueBus events'
16
+ task unsubscribe: [:preload] do
15
17
  manager = ::QueueBus::TaskManager.new(true)
16
18
  count = manager.unsubscribe!
17
- puts "No subscriptions unsubscribed" if count == 0
19
+ puts 'No subscriptions unsubscribed' if count == 0
18
20
  end
19
21
 
20
- desc "List QueueBus queues that need worked"
21
- task :queues => [ :preload ] do
22
+ desc 'List QueueBus queues that need worked'
23
+ task queues: [:preload] do
22
24
  manager = ::QueueBus::TaskManager.new(false)
23
25
  queues = manager.queue_names + ['bus_incoming']
24
- puts queues.join(", ")
26
+ puts queues.join(', ')
25
27
  end
26
28
 
27
- desc "list time based subscriptions"
28
- task :list_scheduled => [ :preload ] do
29
+ desc 'list time based subscriptions'
30
+ task list_scheduled: [:preload] do
29
31
  scheduled_list = QueueBus::Application.all.flat_map do |app|
30
32
  app.send(:subscriptions).all
31
- .select { |s| s.matcher.filters['bus_event_type'] == 'heartbeat_minutes' }
33
+ .select { |s| s.matcher.filters['bus_event_type'] == 'heartbeat_minutes' }
34
+ end
35
+ scheduled_text_list = scheduled_list.collect do |e|
36
+ [e.key, e.matcher.filters['hour'] || '*', e.matcher.filters['minute'] || '*']
32
37
  end
33
- scheduled_text_list = scheduled_list.collect { |e| [ e.key, e.matcher.filters['hour'] || "*", e.matcher.filters['minute'] || "*" ] }
34
38
  puts 'key, hour, minute'
35
39
  puts scheduled_text_list.sort_by { |(_, hour, minute)| [hour.to_i, minute.to_i] }.map(&:to_csv)
36
40
  end
@@ -41,3 +45,4 @@ namespace :queuebus do
41
45
  require 'queue-bus'
42
46
  end
43
47
  end
48
+ # rubocop:enable Metrics/BlockLength
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'multi_json'
2
4
 
3
5
  module QueueBus
@@ -30,14 +32,14 @@ module QueueBus
30
32
  raise DecodeException, e.message, e.backtrace
31
33
  end
32
34
  end
33
-
35
+
34
36
  def underscore(camel_cased_word)
35
37
  word = camel_cased_word.to_s.dup
36
38
  word.gsub!('::', '/')
37
39
  # word.gsub!(/(?:([A-Za-z\d])|^)(#{inflections.acronym_regex})(?=\b|[^a-z])/) { "#{$1}#{$1 && '_'}#{$2.downcase}" }
38
- word.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
39
- word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
40
- word.tr!("-", "_")
40
+ word.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
41
+ word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
42
+ word.tr!('-', '_')
41
43
  word.downcase!
42
44
  word
43
45
  end
@@ -53,11 +55,11 @@ module QueueBus
53
55
  # string = string.sub(/^[a-z\d]*/) { inflections.acronyms[$&] || $&.capitalize }
54
56
  string = string.sub(/^[a-z\d]*/) { $&.capitalize }
55
57
  # string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{inflections.acronyms[$2] || $2.capitalize}" }
56
- string.gsub!(/(?:_|(\/))([a-z\d]*)/i) { "#{$1}#{$2.capitalize}" }
57
- string.gsub!(/\//, '::')
58
+ string.gsub!(%r{(?:_|(/))([a-z\d]*)}i) { "#{Regexp.last_match(1)}#{Regexp.last_match(2).capitalize}" }
59
+ string.gsub!(%r{/}, '::')
58
60
  string
59
61
  end
60
-
62
+
61
63
  def constantize(camel_cased_word)
62
64
  names = camel_cased_word.split('::')
63
65
  names.shift if names.empty? || names.first.empty?
@@ -75,6 +77,7 @@ module QueueBus
75
77
  constant = constant.ancestors.inject do |const, ancestor|
76
78
  break const if ancestor == Object
77
79
  break ancestor if ancestor.const_defined?(name, false)
80
+
78
81
  const
79
82
  end
80
83
 
@@ -84,4 +87,4 @@ module QueueBus
84
87
  end
85
88
  end
86
89
  end
87
- end
90
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
- VERSION = "0.9.0"
4
+ VERSION = '0.9.1'
3
5
  end
@@ -1,11 +1,12 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
4
  class Worker
3
-
4
5
  def self.perform(json)
5
6
  klass = nil
6
7
  attributes = ::QueueBus::Util.decode(json)
7
8
  begin
8
- class_name = attributes["bus_class_proxy"]
9
+ class_name = attributes['bus_class_proxy']
9
10
  klass = ::QueueBus::Util.constantize(class_name)
10
11
  rescue NameError
11
12
  # not there anymore
@@ -1,33 +1,34 @@
1
- # -*- encoding: utf-8 -*-
2
- $:.push File.expand_path("../lib", __FILE__)
3
- require "queue_bus/version"
1
+ # frozen_string_literal: true
2
+
3
+ $:.push File.expand_path('lib', __dir__)
4
+ require 'queue_bus/version'
4
5
 
5
6
  Gem::Specification.new do |s|
6
- s.name = "queue-bus"
7
+ s.name = 'queue-bus'
7
8
  s.version = QueueBus::VERSION
8
- s.authors = ["Brian Leonard"]
9
- s.email = ["brian@bleonard.com"]
10
- s.homepage = ""
11
- s.summary = %q{A simple event bus on top of background queues}
12
- s.description = %q{A simple event bus on top of common background queues. Publish and subscribe to events as they occur using what you already have.}
9
+ s.authors = ['Brian Leonard']
10
+ s.email = ['brian@bleonard.com']
11
+ s.homepage = ''
12
+ s.summary = 'A simple event bus on top of background queues'
13
+ s.description = 'A simple event bus on top of common background queues. Publish and subscribe to events as they occur using what you already have.'
13
14
 
14
- s.rubyforge_project = "queue-bus"
15
+ s.rubyforge_project = 'queue-bus'
15
16
 
16
17
  s.files = `git ls-files`.split("\n")
17
18
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
- s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
- s.require_paths = ["lib"]
19
+ s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
20
+ s.require_paths = ['lib']
20
21
 
21
- s.add_dependency("multi_json")
22
- s.add_dependency("redis")
22
+ s.add_dependency('multi_json')
23
+ s.add_dependency('redis')
23
24
 
24
25
  # if using resque
25
26
  # s.add_development_dependency('resque', ['>= 1.10.0', '< 2.0'])
26
27
  # s.add_development_dependency('resque-scheduler', '>= 2.0.1')
27
28
  # s.add_development_dependency('resque-retry')
28
29
 
29
- s.add_development_dependency("rspec")
30
- s.add_development_dependency("timecop")
31
- s.add_development_dependency("json_pure")
32
- s.add_development_dependency("rubocop")
30
+ s.add_development_dependency('json_pure')
31
+ s.add_development_dependency('rspec')
32
+ s.add_development_dependency('rubocop')
33
+ s.add_development_dependency('timecop')
33
34
  end