queue-bus 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/CHANGELOG.md +9 -0
- data/Gemfile +4 -2
- data/Rakefile +2 -0
- data/lib/queue-bus.rb +15 -12
- data/lib/queue_bus/adapters/base.rb +4 -2
- data/lib/queue_bus/adapters/data.rb +12 -11
- data/lib/queue_bus/application.rb +13 -15
- data/lib/queue_bus/dispatch.rb +14 -12
- data/lib/queue_bus/dispatchers.rb +12 -5
- data/lib/queue_bus/driver.rb +15 -10
- data/lib/queue_bus/heartbeat.rb +32 -30
- data/lib/queue_bus/local.rb +9 -9
- data/lib/queue_bus/matcher.rb +36 -27
- data/lib/queue_bus/publisher.rb +7 -5
- data/lib/queue_bus/publishing.rb +31 -24
- data/lib/queue_bus/rider.rb +26 -22
- data/lib/queue_bus/subscriber.rb +20 -14
- data/lib/queue_bus/subscription.rb +25 -15
- data/lib/queue_bus/subscription_list.rb +30 -12
- data/lib/queue_bus/task_manager.rb +18 -16
- data/lib/queue_bus/tasks.rb +20 -15
- data/lib/queue_bus/util.rb +11 -8
- data/lib/queue_bus/version.rb +3 -1
- data/lib/queue_bus/worker.rb +3 -2
- data/queue-bus.gemspec +19 -18
- data/spec/adapter/publish_at_spec.rb +28 -25
- data/spec/adapter/support.rb +7 -1
- data/spec/adapter_spec.rb +4 -2
- data/spec/application_spec.rb +97 -97
- data/spec/dispatch_spec.rb +48 -51
- data/spec/driver_spec.rb +60 -58
- data/spec/heartbeat_spec.rb +26 -24
- data/spec/integration_spec.rb +41 -40
- data/spec/matcher_spec.rb +104 -102
- data/spec/publish_spec.rb +46 -46
- data/spec/publisher_spec.rb +3 -1
- data/spec/rider_spec.rb +16 -14
- data/spec/spec_helper.rb +2 -2
- data/spec/subscriber_spec.rb +227 -227
- data/spec/subscription_list_spec.rb +31 -31
- data/spec/subscription_spec.rb +37 -36
- data/spec/worker_spec.rb +17 -15
- metadata +7 -6
data/lib/queue_bus/heartbeat.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
|
-
#
|
4
|
+
# When run, will calculate all of the heartbeats that need to be sent and then broadcasts
|
5
|
+
# those events out for execution. By always backfilling it ensures that no heartbeat is
|
6
|
+
# ever missed.
|
3
7
|
class Heartbeat
|
4
|
-
|
5
8
|
class << self
|
6
|
-
|
7
9
|
def lock_key
|
8
|
-
|
10
|
+
'bus:heartbeat:lock'
|
9
11
|
end
|
10
12
|
|
11
13
|
def lock_seconds
|
@@ -38,71 +40,71 @@ module QueueBus
|
|
38
40
|
::QueueBus.redis { |redis| redis.del(lock_key) }
|
39
41
|
end
|
40
42
|
|
41
|
-
|
42
43
|
def redis_key
|
43
|
-
|
44
|
+
'bus:heartbeat:timestamp'
|
44
45
|
end
|
45
46
|
|
46
47
|
def environment_name
|
47
|
-
ENV[
|
48
|
+
ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['BUS_ENV']
|
48
49
|
end
|
49
50
|
|
50
51
|
def get_saved_minute!
|
51
52
|
key = ::QueueBus.redis { |redis| redis.get(redis_key) }
|
52
53
|
return nil if key.nil?
|
54
|
+
|
53
55
|
case environment_name
|
54
56
|
when 'development', 'test'
|
55
57
|
# only 3 minutes in development; otherwise, TONS of events if not run in a while
|
56
|
-
three_ago = Time.now.to_i - 3
|
58
|
+
three_ago = Time.now.to_i / 60 - 3
|
57
59
|
key = three_ago if key.to_i < three_ago
|
58
60
|
end
|
59
|
-
|
61
|
+
key.to_i
|
60
62
|
end
|
61
63
|
|
62
64
|
def set_saved_minute!(epoch_minute)
|
63
65
|
::QueueBus.redis { |redis| redis.set(redis_key, epoch_minute) }
|
64
66
|
end
|
65
67
|
|
66
|
-
def perform(*
|
68
|
+
def perform(*_args)
|
67
69
|
real_now = Time.now.to_i
|
68
70
|
run_until = lock! - 2
|
69
71
|
return if run_until < real_now
|
70
72
|
|
71
|
-
while(
|
73
|
+
while (real_now = Time.now.to_i) < run_until
|
72
74
|
minutes = real_now.to_i / 60
|
73
75
|
last = get_saved_minute!
|
74
76
|
if last
|
75
77
|
break if minutes <= last
|
78
|
+
|
76
79
|
minutes = last + 1
|
77
80
|
end
|
78
81
|
|
79
|
-
seconds = minutes *
|
80
|
-
hours = minutes /
|
81
|
-
days = minutes / (60*24)
|
82
|
+
seconds = minutes * 60
|
83
|
+
hours = minutes / 60
|
84
|
+
days = minutes / (60 * 24)
|
82
85
|
|
83
|
-
now
|
86
|
+
now = Time.at(seconds)
|
84
87
|
|
85
88
|
attributes = {}
|
86
|
-
attributes[
|
87
|
-
attributes[
|
88
|
-
attributes[
|
89
|
-
attributes[
|
90
|
-
|
91
|
-
attributes[
|
92
|
-
attributes[
|
93
|
-
attributes[
|
94
|
-
attributes[
|
95
|
-
attributes[
|
96
|
-
attributes[
|
97
|
-
attributes[
|
98
|
-
|
99
|
-
::QueueBus.publish(
|
89
|
+
attributes['epoch_seconds'] = seconds
|
90
|
+
attributes['epoch_minutes'] = minutes
|
91
|
+
attributes['epoch_hours'] = hours
|
92
|
+
attributes['epoch_days'] = days
|
93
|
+
|
94
|
+
attributes['minute'] = now.min
|
95
|
+
attributes['hour'] = now.hour
|
96
|
+
attributes['day'] = now.day
|
97
|
+
attributes['month'] = now.month
|
98
|
+
attributes['year'] = now.year
|
99
|
+
attributes['yday'] = now.yday
|
100
|
+
attributes['wday'] = now.wday
|
101
|
+
|
102
|
+
::QueueBus.publish('heartbeat_minutes', attributes)
|
100
103
|
set_saved_minute!(minutes)
|
101
104
|
end
|
102
105
|
|
103
106
|
unlock!
|
104
107
|
end
|
105
108
|
end
|
106
|
-
|
107
109
|
end
|
108
110
|
end
|
data/lib/queue_bus/local.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
4
|
# only process local queues
|
3
5
|
class Local
|
4
|
-
|
5
6
|
class << self
|
6
7
|
def publish(attributes = {})
|
7
8
|
if ::QueueBus.local_mode == :suppress
|
8
9
|
::QueueBus.log_worker("Suppressed: #{attributes.inspect}")
|
9
|
-
return
|
10
|
+
return # not doing anything
|
10
11
|
end
|
11
12
|
|
12
13
|
# To json and back to simlulate enqueueing
|
@@ -17,15 +18,15 @@ module QueueBus
|
|
17
18
|
|
18
19
|
# looking for subscriptions, not queues
|
19
20
|
subscription_matches(attributes).each do |sub|
|
20
|
-
bus_attr = {
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
21
|
+
bus_attr = { 'bus_driven_at' => Time.now.to_i,
|
22
|
+
'bus_rider_queue' => sub.queue_name,
|
23
|
+
'bus_rider_app_key' => sub.app_key,
|
24
|
+
'bus_rider_sub_key' => sub.key,
|
25
|
+
'bus_rider_class_name' => sub.class_name }
|
25
26
|
to_publish = bus_attr.merge(attributes || {})
|
26
27
|
if ::QueueBus.local_mode == :standalone
|
27
28
|
::QueueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr.merge(attributes || {}))
|
28
|
-
else
|
29
|
+
else # defaults to inline mode
|
29
30
|
sub.execute!(to_publish)
|
30
31
|
end
|
31
32
|
end
|
@@ -41,6 +42,5 @@ module QueueBus
|
|
41
42
|
out
|
42
43
|
end
|
43
44
|
end
|
44
|
-
|
45
45
|
end
|
46
46
|
end
|
data/lib/queue_bus/matcher.rb
CHANGED
@@ -1,70 +1,80 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# Tests whether a field on a bus event matches a filter.
|
2
5
|
class Matcher
|
3
|
-
SPECIAL_PREPEND =
|
6
|
+
SPECIAL_PREPEND = 'bus_special_value_'
|
4
7
|
attr_reader :filters
|
5
8
|
def initialize(hash)
|
6
9
|
@filters = encode(hash)
|
7
10
|
end
|
8
|
-
|
11
|
+
|
9
12
|
def to_redis
|
10
13
|
@filters
|
11
14
|
end
|
12
|
-
|
15
|
+
|
13
16
|
def match?(attribute_name, attributes)
|
14
17
|
mine = filters[attribute_name].to_s
|
15
|
-
return false if mine.
|
16
|
-
|
18
|
+
return false if mine.empty?
|
19
|
+
|
17
20
|
given = attributes[attribute_name]
|
18
21
|
case mine
|
19
22
|
when "#{SPECIAL_PREPEND}key"
|
20
|
-
return true if attributes.
|
23
|
+
return true if attributes.key?(attribute_name)
|
24
|
+
|
21
25
|
return false
|
22
26
|
when "#{SPECIAL_PREPEND}blank"
|
23
|
-
return true if given.to_s.strip.
|
27
|
+
return true if given.to_s.strip.empty?
|
28
|
+
|
24
29
|
return false
|
25
30
|
when "#{SPECIAL_PREPEND}empty"
|
26
|
-
return false if given
|
27
|
-
return true if given.to_s.
|
31
|
+
return false if given.nil?
|
32
|
+
return true if given.to_s.empty?
|
33
|
+
|
28
34
|
return false
|
29
35
|
when "#{SPECIAL_PREPEND}nil"
|
30
|
-
return true if given
|
36
|
+
return true if given.nil?
|
37
|
+
|
31
38
|
return false
|
32
39
|
when "#{SPECIAL_PREPEND}value"
|
33
|
-
return false if given
|
40
|
+
return false if given.nil?
|
41
|
+
|
34
42
|
return true
|
35
43
|
when "#{SPECIAL_PREPEND}present"
|
36
|
-
return true
|
44
|
+
return true unless given.to_s.strip.empty?
|
45
|
+
|
37
46
|
return false
|
38
47
|
end
|
39
|
-
|
48
|
+
|
40
49
|
given = given.to_s
|
41
|
-
|
50
|
+
|
42
51
|
return true if mine == given
|
52
|
+
|
43
53
|
begin
|
44
54
|
# if it's already a regex, don't mess with it
|
45
55
|
# otherwise, it should have start and end line situation
|
46
|
-
if mine[0..6] ==
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
56
|
+
regex = if mine[0..6] == '(?-mix:'
|
57
|
+
Regexp.new(mine)
|
58
|
+
else
|
59
|
+
Regexp.new("^#{mine}$")
|
60
|
+
end
|
51
61
|
return !!regex.match(given)
|
52
|
-
rescue
|
62
|
+
rescue StandardError
|
53
63
|
return false
|
54
64
|
end
|
55
65
|
end
|
56
|
-
|
66
|
+
|
57
67
|
def matches?(attributes)
|
58
68
|
return false if filters.empty?
|
59
|
-
return false if attributes
|
60
|
-
|
69
|
+
return false if attributes.nil?
|
70
|
+
|
61
71
|
filters.keys.each do |key|
|
62
72
|
return false unless match?(key, attributes)
|
63
73
|
end
|
64
|
-
|
74
|
+
|
65
75
|
true
|
66
76
|
end
|
67
|
-
|
77
|
+
|
68
78
|
def encode(hash)
|
69
79
|
out = {}
|
70
80
|
hash.each do |key, value|
|
@@ -73,9 +83,8 @@ module QueueBus
|
|
73
83
|
value = "#{SPECIAL_PREPEND}#{value}"
|
74
84
|
end
|
75
85
|
out[key.to_s] = value.to_s
|
76
|
-
end
|
86
|
+
end
|
77
87
|
out
|
78
88
|
end
|
79
89
|
end
|
80
90
|
end
|
81
|
-
|
data/lib/queue_bus/publisher.rb
CHANGED
@@ -1,14 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
|
-
#
|
4
|
+
# A simple publishing worker for QueueBus. Makes publishing asynchronously possible since
|
5
|
+
# it may be enqueued to the background worker with a delay. This will allow the event to
|
6
|
+
# be published at a later time.
|
3
7
|
class Publisher
|
4
|
-
|
5
8
|
class << self
|
6
9
|
def perform(attributes)
|
7
|
-
event_type = attributes[
|
10
|
+
event_type = attributes['bus_event_type']
|
8
11
|
::QueueBus.log_worker("Publisher running: #{event_type} - #{attributes.inspect}")
|
9
12
|
::QueueBus.publish(event_type, attributes)
|
10
13
|
end
|
11
14
|
end
|
12
|
-
|
13
15
|
end
|
14
|
-
end
|
16
|
+
end
|
data/lib/queue_bus/publishing.rb
CHANGED
@@ -1,18 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# The publishing mixin provides the main interactions that users will use
|
5
|
+
# to interact with the queue bus. This module is not interacted with directly and instead
|
6
|
+
# is included inte the `QueueBus` module.
|
2
7
|
module Publishing
|
3
|
-
|
4
8
|
def with_global_attributes(attributes)
|
5
9
|
original_timezone = false
|
6
10
|
original_locale = false
|
7
11
|
|
8
|
-
if attributes[
|
12
|
+
if attributes['bus_locale'] && defined?(I18n) && I18n.respond_to?(:locale=)
|
9
13
|
original_locale = I18n.locale if I18n.respond_to?(:locale)
|
10
|
-
I18n.locale = attributes[
|
14
|
+
I18n.locale = attributes['bus_locale']
|
11
15
|
end
|
12
16
|
|
13
|
-
if attributes[
|
17
|
+
if attributes['bus_timezone'] && defined?(Time) && Time.respond_to?(:zone=)
|
14
18
|
original_timezone = Time.zone if Time.respond_to?(:zone)
|
15
|
-
Time.zone = attributes[
|
19
|
+
Time.zone = attributes['bus_timezone']
|
16
20
|
end
|
17
21
|
|
18
22
|
yield
|
@@ -21,13 +25,17 @@ module QueueBus
|
|
21
25
|
Time.zone = original_timezone unless original_timezone == false
|
22
26
|
end
|
23
27
|
|
24
|
-
def publish_metadata(event_type, attributes={})
|
28
|
+
def publish_metadata(event_type, attributes = {})
|
25
29
|
# TODO: "bus_app_key" => application.app_key ?
|
26
|
-
bus_attr = {
|
27
|
-
bus_attr[
|
28
|
-
bus_attr[
|
29
|
-
|
30
|
-
|
30
|
+
bus_attr = { 'bus_published_at' => Time.now.to_i, 'bus_event_type' => event_type }
|
31
|
+
bus_attr['bus_id'] = "#{Time.now.to_i}-#{generate_uuid}"
|
32
|
+
bus_attr['bus_app_hostname'] = ::QueueBus.hostname
|
33
|
+
if defined?(I18n) && I18n.respond_to?(:locale) && I18n.locale
|
34
|
+
bus_attr['bus_locale'] = I18n.locale.to_s
|
35
|
+
end
|
36
|
+
if defined?(Time) && Time.respond_to?(:zone) && Time.zone
|
37
|
+
bus_attr['bus_timezone'] = Time.zone.name
|
38
|
+
end
|
31
39
|
out = bus_attr.merge(attributes || {})
|
32
40
|
::QueueBus.before_publish_callback(out)
|
33
41
|
out
|
@@ -35,15 +43,14 @@ module QueueBus
|
|
35
43
|
|
36
44
|
def generate_uuid
|
37
45
|
require 'securerandom' unless defined?(SecureRandom)
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
return "#{rand(max)}-#{rand(max)}-#{rand(max)}"
|
46
|
+
SecureRandom.uuid
|
47
|
+
rescue Exception => e
|
48
|
+
# secure random not there
|
49
|
+
# big random number a few times
|
50
|
+
n_bytes = [42].pack('i').size
|
51
|
+
n_bits = n_bytes * 8
|
52
|
+
max = 2**(n_bits - 2) - 1
|
53
|
+
"#{rand(max)}-#{rand(max)}-#{rand(max)}"
|
47
54
|
end
|
48
55
|
|
49
56
|
def publish(event_type, attributes = {})
|
@@ -59,9 +66,9 @@ module QueueBus
|
|
59
66
|
|
60
67
|
def publish_at(timestamp_or_epoch, event_type, attributes = {})
|
61
68
|
to_publish = publish_metadata(event_type, attributes)
|
62
|
-
to_publish[
|
63
|
-
to_publish.delete(
|
64
|
-
to_publish[
|
69
|
+
to_publish['bus_delayed_until'] ||= timestamp_or_epoch.to_i
|
70
|
+
to_publish.delete('bus_published_at') unless attributes['bus_published_at'] # will be put on when it actually does it
|
71
|
+
to_publish['bus_class_proxy'] = ::QueueBus::Publisher.name.to_s
|
65
72
|
|
66
73
|
::QueueBus.log_application("Event published:#{event_type} #{to_publish.inspect} publish_at: #{timestamp_or_epoch.to_i}")
|
67
74
|
delayed_enqueue_to(timestamp_or_epoch.to_i, incoming_queue, ::QueueBus::Worker, to_publish)
|
@@ -69,7 +76,7 @@ module QueueBus
|
|
69
76
|
|
70
77
|
def enqueue_to(queue_name, class_name, hash)
|
71
78
|
class_name = class_name.name if class_name.is_a?(Class)
|
72
|
-
hash = hash.merge(
|
79
|
+
hash = hash.merge('bus_class_proxy' => class_name.to_s)
|
73
80
|
::QueueBus.adapter.enqueue(queue_name, ::QueueBus::Worker, ::QueueBus::Util.encode(hash || {}))
|
74
81
|
end
|
75
82
|
|
data/lib/queue_bus/rider.rb
CHANGED
@@ -1,27 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
|
-
#
|
4
|
+
# The Rider is meant to execute subscriptions.
|
5
|
+
#
|
6
|
+
# When your application has subscriptions to an event we still need to enqueue some executor that
|
7
|
+
# will execute the block that was registered. The Rider will effectively ride the bus for your
|
8
|
+
# subscribed event. One Rider is launched for each subscription on an event.
|
3
9
|
class Rider
|
10
|
+
def self.perform(attributes = {})
|
11
|
+
sub_key = attributes['bus_rider_sub_key']
|
12
|
+
app_key = attributes['bus_rider_app_key']
|
13
|
+
raise 'No application key passed' if app_key.to_s == ''
|
14
|
+
raise 'No subcription key passed' if sub_key.to_s == ''
|
15
|
+
|
16
|
+
attributes ||= {}
|
17
|
+
|
18
|
+
::QueueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
|
19
|
+
|
20
|
+
# attributes that should be available
|
21
|
+
# attributes["bus_event_type"]
|
22
|
+
# attributes["bus_app_key"]
|
23
|
+
# attributes["bus_published_at"]
|
24
|
+
# attributes["bus_driven_at"]
|
4
25
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
app_key = attributes["bus_rider_app_key"]
|
9
|
-
raise "No application key passed" if app_key.to_s == ""
|
10
|
-
raise "No subcription key passed" if sub_key.to_s == ""
|
11
|
-
|
12
|
-
attributes ||= {}
|
13
|
-
|
14
|
-
::QueueBus.log_worker("Rider received: #{app_key} #{sub_key} #{attributes.inspect}")
|
15
|
-
|
16
|
-
# attributes that should be available
|
17
|
-
# attributes["bus_event_type"]
|
18
|
-
# attributes["bus_app_key"]
|
19
|
-
# attributes["bus_published_at"]
|
20
|
-
# attributes["bus_driven_at"]
|
21
|
-
|
22
|
-
# (now running with the real app that subscribed)
|
23
|
-
::QueueBus.dispatcher_execute(app_key, sub_key, attributes.merge("bus_executed_at" => Time.now.to_i))
|
24
|
-
end
|
26
|
+
# (now running with the real app that subscribed)
|
27
|
+
::QueueBus.dispatcher_execute(app_key, sub_key,
|
28
|
+
attributes.merge('bus_executed_at' => Time.now.to_i))
|
25
29
|
end
|
26
30
|
end
|
27
|
-
end
|
31
|
+
end
|