queue-bus 0.9.0 → 0.9.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +21 -0
  3. data/CHANGELOG.md +9 -0
  4. data/Gemfile +4 -2
  5. data/Rakefile +2 -0
  6. data/lib/queue-bus.rb +15 -12
  7. data/lib/queue_bus/adapters/base.rb +4 -2
  8. data/lib/queue_bus/adapters/data.rb +12 -11
  9. data/lib/queue_bus/application.rb +13 -15
  10. data/lib/queue_bus/dispatch.rb +14 -12
  11. data/lib/queue_bus/dispatchers.rb +12 -5
  12. data/lib/queue_bus/driver.rb +15 -10
  13. data/lib/queue_bus/heartbeat.rb +32 -30
  14. data/lib/queue_bus/local.rb +9 -9
  15. data/lib/queue_bus/matcher.rb +36 -27
  16. data/lib/queue_bus/publisher.rb +7 -5
  17. data/lib/queue_bus/publishing.rb +31 -24
  18. data/lib/queue_bus/rider.rb +26 -22
  19. data/lib/queue_bus/subscriber.rb +20 -14
  20. data/lib/queue_bus/subscription.rb +25 -15
  21. data/lib/queue_bus/subscription_list.rb +30 -12
  22. data/lib/queue_bus/task_manager.rb +18 -16
  23. data/lib/queue_bus/tasks.rb +20 -15
  24. data/lib/queue_bus/util.rb +11 -8
  25. data/lib/queue_bus/version.rb +3 -1
  26. data/lib/queue_bus/worker.rb +3 -2
  27. data/queue-bus.gemspec +19 -18
  28. data/spec/adapter/publish_at_spec.rb +28 -25
  29. data/spec/adapter/support.rb +7 -1
  30. data/spec/adapter_spec.rb +4 -2
  31. data/spec/application_spec.rb +97 -97
  32. data/spec/dispatch_spec.rb +48 -51
  33. data/spec/driver_spec.rb +60 -58
  34. data/spec/heartbeat_spec.rb +26 -24
  35. data/spec/integration_spec.rb +41 -40
  36. data/spec/matcher_spec.rb +104 -102
  37. data/spec/publish_spec.rb +46 -46
  38. data/spec/publisher_spec.rb +3 -1
  39. data/spec/rider_spec.rb +16 -14
  40. data/spec/spec_helper.rb +2 -2
  41. data/spec/subscriber_spec.rb +227 -227
  42. data/spec/subscription_list_spec.rb +31 -31
  43. data/spec/subscription_spec.rb +37 -36
  44. data/spec/worker_spec.rb +17 -15
  45. metadata +7 -6
@@ -1,11 +1,13 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
- # publishes event about the current time
4
+ # When run, will calculate all of the heartbeats that need to be sent and then broadcasts
5
+ # those events out for execution. By always backfilling it ensures that no heartbeat is
6
+ # ever missed.
3
7
  class Heartbeat
4
-
5
8
  class << self
6
-
7
9
  def lock_key
8
- "bus:heartbeat:lock"
10
+ 'bus:heartbeat:lock'
9
11
  end
10
12
 
11
13
  def lock_seconds
@@ -38,71 +40,71 @@ module QueueBus
38
40
  ::QueueBus.redis { |redis| redis.del(lock_key) }
39
41
  end
40
42
 
41
-
42
43
  def redis_key
43
- "bus:heartbeat:timestamp"
44
+ 'bus:heartbeat:timestamp'
44
45
  end
45
46
 
46
47
  def environment_name
47
- ENV["RAILS_ENV"] || ENV["RACK_ENV"] || ENV["BUS_ENV"]
48
+ ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['BUS_ENV']
48
49
  end
49
50
 
50
51
  def get_saved_minute!
51
52
  key = ::QueueBus.redis { |redis| redis.get(redis_key) }
52
53
  return nil if key.nil?
54
+
53
55
  case environment_name
54
56
  when 'development', 'test'
55
57
  # only 3 minutes in development; otherwise, TONS of events if not run in a while
56
- three_ago = Time.now.to_i - 3*60*60
58
+ three_ago = Time.now.to_i / 60 - 3
57
59
  key = three_ago if key.to_i < three_ago
58
60
  end
59
- return key.to_i
61
+ key.to_i
60
62
  end
61
63
 
62
64
  def set_saved_minute!(epoch_minute)
63
65
  ::QueueBus.redis { |redis| redis.set(redis_key, epoch_minute) }
64
66
  end
65
67
 
66
- def perform(*args)
68
+ def perform(*_args)
67
69
  real_now = Time.now.to_i
68
70
  run_until = lock! - 2
69
71
  return if run_until < real_now
70
72
 
71
- while((real_now = Time.now.to_i) < run_until)
73
+ while (real_now = Time.now.to_i) < run_until
72
74
  minutes = real_now.to_i / 60
73
75
  last = get_saved_minute!
74
76
  if last
75
77
  break if minutes <= last
78
+
76
79
  minutes = last + 1
77
80
  end
78
81
 
79
- seconds = minutes * (60)
80
- hours = minutes / (60)
81
- days = minutes / (60*24)
82
+ seconds = minutes * 60
83
+ hours = minutes / 60
84
+ days = minutes / (60 * 24)
82
85
 
83
- now = Time.at(seconds)
86
+ now = Time.at(seconds)
84
87
 
85
88
  attributes = {}
86
- attributes["epoch_seconds"] = seconds
87
- attributes["epoch_minutes"] = minutes
88
- attributes["epoch_hours"] = hours
89
- attributes["epoch_days"] = days
90
-
91
- attributes["minute"] = now.min
92
- attributes["hour"] = now.hour
93
- attributes["day"] = now.day
94
- attributes["month"] = now.month
95
- attributes["year"] = now.year
96
- attributes["yday"] = now.yday
97
- attributes["wday"] = now.wday
98
-
99
- ::QueueBus.publish("heartbeat_minutes", attributes)
89
+ attributes['epoch_seconds'] = seconds
90
+ attributes['epoch_minutes'] = minutes
91
+ attributes['epoch_hours'] = hours
92
+ attributes['epoch_days'] = days
93
+
94
+ attributes['minute'] = now.min
95
+ attributes['hour'] = now.hour
96
+ attributes['day'] = now.day
97
+ attributes['month'] = now.month
98
+ attributes['year'] = now.year
99
+ attributes['yday'] = now.yday
100
+ attributes['wday'] = now.wday
101
+
102
+ ::QueueBus.publish('heartbeat_minutes', attributes)
100
103
  set_saved_minute!(minutes)
101
104
  end
102
105
 
103
106
  unlock!
104
107
  end
105
108
  end
106
-
107
109
  end
108
110
  end
@@ -1,12 +1,13 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
4
  # only process local queues
3
5
  class Local
4
-
5
6
  class << self
6
7
  def publish(attributes = {})
7
8
  if ::QueueBus.local_mode == :suppress
8
9
  ::QueueBus.log_worker("Suppressed: #{attributes.inspect}")
9
- return # not doing anything
10
+ return # not doing anything
10
11
  end
11
12
 
12
13
  # To json and back to simlulate enqueueing
@@ -17,15 +18,15 @@ module QueueBus
17
18
 
18
19
  # looking for subscriptions, not queues
19
20
  subscription_matches(attributes).each do |sub|
20
- bus_attr = { "bus_driven_at" => Time.now.to_i,
21
- "bus_rider_queue" => sub.queue_name,
22
- "bus_rider_app_key" => sub.app_key,
23
- "bus_rider_sub_key" => sub.key,
24
- "bus_rider_class_name" => sub.class_name}
21
+ bus_attr = { 'bus_driven_at' => Time.now.to_i,
22
+ 'bus_rider_queue' => sub.queue_name,
23
+ 'bus_rider_app_key' => sub.app_key,
24
+ 'bus_rider_sub_key' => sub.key,
25
+ 'bus_rider_class_name' => sub.class_name }
25
26
  to_publish = bus_attr.merge(attributes || {})
26
27
  if ::QueueBus.local_mode == :standalone
27
28
  ::QueueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
28
- else # defaults to inline mode
29
+ else # defaults to inline mode
29
30
  sub.execute!(to_publish)
30
31
  end
31
32
  end
@@ -41,6 +42,5 @@ module QueueBus
41
42
  out
42
43
  end
43
44
  end
44
-
45
45
  end
46
46
  end
@@ -1,70 +1,80 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # Tests whether a field on a bus event matches a filter.
2
5
  class Matcher
3
- SPECIAL_PREPEND = "bus_special_value_"
6
+ SPECIAL_PREPEND = 'bus_special_value_'
4
7
  attr_reader :filters
5
8
  def initialize(hash)
6
9
  @filters = encode(hash)
7
10
  end
8
-
11
+
9
12
  def to_redis
10
13
  @filters
11
14
  end
12
-
15
+
13
16
  def match?(attribute_name, attributes)
14
17
  mine = filters[attribute_name].to_s
15
- return false if mine.size == 0
16
-
18
+ return false if mine.empty?
19
+
17
20
  given = attributes[attribute_name]
18
21
  case mine
19
22
  when "#{SPECIAL_PREPEND}key"
20
- return true if attributes.has_key?(attribute_name)
23
+ return true if attributes.key?(attribute_name)
24
+
21
25
  return false
22
26
  when "#{SPECIAL_PREPEND}blank"
23
- return true if given.to_s.strip.size == 0
27
+ return true if given.to_s.strip.empty?
28
+
24
29
  return false
25
30
  when "#{SPECIAL_PREPEND}empty"
26
- return false if given == nil
27
- return true if given.to_s.size == 0
31
+ return false if given.nil?
32
+ return true if given.to_s.empty?
33
+
28
34
  return false
29
35
  when "#{SPECIAL_PREPEND}nil"
30
- return true if given == nil
36
+ return true if given.nil?
37
+
31
38
  return false
32
39
  when "#{SPECIAL_PREPEND}value"
33
- return false if given == nil
40
+ return false if given.nil?
41
+
34
42
  return true
35
43
  when "#{SPECIAL_PREPEND}present"
36
- return true if given.to_s.strip.size > 0
44
+ return true unless given.to_s.strip.empty?
45
+
37
46
  return false
38
47
  end
39
-
48
+
40
49
  given = given.to_s
41
-
50
+
42
51
  return true if mine == given
52
+
43
53
  begin
44
54
  # if it's already a regex, don't mess with it
45
55
  # otherwise, it should have start and end line situation
46
- if mine[0..6] == "(?-mix:"
47
- regex = Regexp.new(mine)
48
- else
49
- regex = Regexp.new("^#{mine}$")
50
- end
56
+ regex = if mine[0..6] == '(?-mix:'
57
+ Regexp.new(mine)
58
+ else
59
+ Regexp.new("^#{mine}$")
60
+ end
51
61
  return !!regex.match(given)
52
- rescue
62
+ rescue StandardError
53
63
  return false
54
64
  end
55
65
  end
56
-
66
+
57
67
  def matches?(attributes)
58
68
  return false if filters.empty?
59
- return false if attributes == nil
60
-
69
+ return false if attributes.nil?
70
+
61
71
  filters.keys.each do |key|
62
72
  return false unless match?(key, attributes)
63
73
  end
64
-
74
+
65
75
  true
66
76
  end
67
-
77
+
68
78
  def encode(hash)
69
79
  out = {}
70
80
  hash.each do |key, value|
@@ -73,9 +83,8 @@ module QueueBus
73
83
  value = "#{SPECIAL_PREPEND}#{value}"
74
84
  end
75
85
  out[key.to_s] = value.to_s
76
- end
86
+ end
77
87
  out
78
88
  end
79
89
  end
80
90
  end
81
-
@@ -1,14 +1,16 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
- # publishes on a delay
4
+ # A simple publishing worker for QueueBus. Makes publishing asynchronously possible since
5
+ # it may be enqueued to the background worker with a delay. This will allow the event to
6
+ # be published at a later time.
3
7
  class Publisher
4
-
5
8
  class << self
6
9
  def perform(attributes)
7
- event_type = attributes["bus_event_type"]
10
+ event_type = attributes['bus_event_type']
8
11
  ::QueueBus.log_worker("Publisher running: #{event_type} - #{attributes.inspect}")
9
12
  ::QueueBus.publish(event_type, attributes)
10
13
  end
11
14
  end
12
-
13
15
  end
14
- end
16
+ end
@@ -1,18 +1,22 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # The publishing mixin provides the main interactions that users will use
5
+ # to interact with the queue bus. This module is not interacted with directly and instead
6
+ # is included inte the `QueueBus` module.
2
7
  module Publishing
3
-
4
8
  def with_global_attributes(attributes)
5
9
  original_timezone = false
6
10
  original_locale = false
7
11
 
8
- if attributes["bus_locale"] && defined?(I18n) && I18n.respond_to?(:locale=)
12
+ if attributes['bus_locale'] && defined?(I18n) && I18n.respond_to?(:locale=)
9
13
  original_locale = I18n.locale if I18n.respond_to?(:locale)
10
- I18n.locale = attributes["bus_locale"]
14
+ I18n.locale = attributes['bus_locale']
11
15
  end
12
16
 
13
- if attributes["bus_timezone"] && defined?(Time) && Time.respond_to?(:zone=)
17
+ if attributes['bus_timezone'] && defined?(Time) && Time.respond_to?(:zone=)
14
18
  original_timezone = Time.zone if Time.respond_to?(:zone)
15
- Time.zone = attributes["bus_timezone"]
19
+ Time.zone = attributes['bus_timezone']
16
20
  end
17
21
 
18
22
  yield
@@ -21,13 +25,17 @@ module QueueBus
21
25
  Time.zone = original_timezone unless original_timezone == false
22
26
  end
23
27
 
24
- def publish_metadata(event_type, attributes={})
28
+ def publish_metadata(event_type, attributes = {})
25
29
  # TODO: "bus_app_key" => application.app_key ?
26
- bus_attr = {"bus_published_at" => Time.now.to_i, "bus_event_type" => event_type}
27
- bus_attr["bus_id"] = "#{Time.now.to_i}-#{generate_uuid}"
28
- bus_attr["bus_app_hostname"] = ::QueueBus.hostname
29
- bus_attr["bus_locale"] = I18n.locale.to_s if defined?(I18n) && I18n.respond_to?(:locale) && I18n.locale
30
- bus_attr["bus_timezone"] = Time.zone.name if defined?(Time) && Time.respond_to?(:zone) && Time.zone
30
+ bus_attr = { 'bus_published_at' => Time.now.to_i, 'bus_event_type' => event_type }
31
+ bus_attr['bus_id'] = "#{Time.now.to_i}-#{generate_uuid}"
32
+ bus_attr['bus_app_hostname'] = ::QueueBus.hostname
33
+ if defined?(I18n) && I18n.respond_to?(:locale) && I18n.locale
34
+ bus_attr['bus_locale'] = I18n.locale.to_s
35
+ end
36
+ if defined?(Time) && Time.respond_to?(:zone) && Time.zone
37
+ bus_attr['bus_timezone'] = Time.zone.name
38
+ end
31
39
  out = bus_attr.merge(attributes || {})
32
40
  ::QueueBus.before_publish_callback(out)
33
41
  out
@@ -35,15 +43,14 @@ module QueueBus
35
43
 
36
44
  def generate_uuid
37
45
  require 'securerandom' unless defined?(SecureRandom)
38
- return SecureRandom.uuid
39
-
40
- rescue Exception => e
41
- # secure random not there
42
- # big random number a few times
43
- n_bytes = [42].pack('i').size
44
- n_bits = n_bytes * 8
45
- max = 2 ** (n_bits - 2) - 1
46
- return "#{rand(max)}-#{rand(max)}-#{rand(max)}"
46
+ SecureRandom.uuid
47
+ rescue Exception => e
48
+ # secure random not there
49
+ # big random number a few times
50
+ n_bytes = [42].pack('i').size
51
+ n_bits = n_bytes * 8
52
+ max = 2**(n_bits - 2) - 1
53
+ "#{rand(max)}-#{rand(max)}-#{rand(max)}"
47
54
  end
48
55
 
49
56
  def publish(event_type, attributes = {})
@@ -59,9 +66,9 @@ module QueueBus
59
66
 
60
67
  def publish_at(timestamp_or_epoch, event_type, attributes = {})
61
68
  to_publish = publish_metadata(event_type, attributes)
62
- to_publish["bus_delayed_until"] ||= timestamp_or_epoch.to_i
63
- to_publish.delete("bus_published_at") unless attributes["bus_published_at"] # will be put on when it actually does it
64
- to_publish["bus_class_proxy"] = ::QueueBus::Publisher.name.to_s
69
+ to_publish['bus_delayed_until'] ||= timestamp_or_epoch.to_i
70
+ to_publish.delete('bus_published_at') unless attributes['bus_published_at'] # will be put on when it actually does it
71
+ to_publish['bus_class_proxy'] = ::QueueBus::Publisher.name.to_s
65
72
 
66
73
  ::QueueBus.log_application("Event published:#{event_type} #{to_publish.inspect} publish_at: #{timestamp_or_epoch.to_i}")
67
74
  delayed_enqueue_to(timestamp_or_epoch.to_i, incoming_queue, ::QueueBus::Worker, to_publish)
@@ -69,7 +76,7 @@ module QueueBus
69
76
 
70
77
  def enqueue_to(queue_name, class_name, hash)
71
78
  class_name = class_name.name if class_name.is_a?(Class)
72
- hash = hash.merge("bus_class_proxy" => class_name.to_s)
79
+ hash = hash.merge('bus_class_proxy' => class_name.to_s)
73
80
  ::QueueBus.adapter.enqueue(queue_name, ::QueueBus::Worker, ::QueueBus::Util.encode(hash || {}))
74
81
  end
75
82
 
@@ -1,27 +1,31 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
- # queue'd in each
4
+ # The Rider is meant to execute subscriptions.
5
+ #
6
+ # When your application has subscriptions to an event we still need to enqueue some executor that
7
+ # will execute the block that was registered. The Rider will effectively ride the bus for your
8
+ # subscribed event. One Rider is launched for each subscription on an event.
3
9
  class Rider
10
+ def self.perform(attributes = {})
11
+ sub_key = attributes['bus_rider_sub_key']
12
+ app_key = attributes['bus_rider_app_key']
13
+ raise 'No application key passed' if app_key.to_s == ''
14
+ raise 'No subcription key passed' if sub_key.to_s == ''
15
+
16
+ attributes ||= {}
17
+
18
+ ::QueueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
19
+
20
+ # attributes that should be available
21
+ # attributes["bus_event_type"]
22
+ # attributes["bus_app_key"]
23
+ # attributes["bus_published_at"]
24
+ # attributes["bus_driven_at"]
4
25
 
5
- class << self
6
- def perform(attributes = {})
7
- sub_key = attributes["bus_rider_sub_key"]
8
- app_key = attributes["bus_rider_app_key"]
9
- raise "No application key passed" if app_key.to_s == ""
10
- raise "No subcription key passed" if sub_key.to_s == ""
11
-
12
- attributes ||= {}
13
-
14
- ::QueueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
15
-
16
- # attributes that should be available
17
- # attributes["bus_event_type"]
18
- # attributes["bus_app_key"]
19
- # attributes["bus_published_at"]
20
- # attributes["bus_driven_at"]
21
-
22
- # (now running with the real app that subscribed)
23
- ::QueueBus.dispatcher_execute(app_key, sub_key, attributes.merge("bus_executed_at" => Time.now.to_i))
24
- end
26
+ # (now running with the real app that subscribed)
27
+ ::QueueBus.dispatcher_execute(app_key, sub_key,
28
+ attributes.merge('bus_executed_at' => Time.now.to_i))
25
29
  end
26
30
  end
27
- end
31
+ end