queue-bus 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +21 -0
- data/CHANGELOG.md +9 -0
- data/Gemfile +4 -2
- data/Rakefile +2 -0
- data/lib/queue-bus.rb +15 -12
- data/lib/queue_bus/adapters/base.rb +4 -2
- data/lib/queue_bus/adapters/data.rb +12 -11
- data/lib/queue_bus/application.rb +13 -15
- data/lib/queue_bus/dispatch.rb +14 -12
- data/lib/queue_bus/dispatchers.rb +12 -5
- data/lib/queue_bus/driver.rb +15 -10
- data/lib/queue_bus/heartbeat.rb +32 -30
- data/lib/queue_bus/local.rb +9 -9
- data/lib/queue_bus/matcher.rb +36 -27
- data/lib/queue_bus/publisher.rb +7 -5
- data/lib/queue_bus/publishing.rb +31 -24
- data/lib/queue_bus/rider.rb +26 -22
- data/lib/queue_bus/subscriber.rb +20 -14
- data/lib/queue_bus/subscription.rb +25 -15
- data/lib/queue_bus/subscription_list.rb +30 -12
- data/lib/queue_bus/task_manager.rb +18 -16
- data/lib/queue_bus/tasks.rb +20 -15
- data/lib/queue_bus/util.rb +11 -8
- data/lib/queue_bus/version.rb +3 -1
- data/lib/queue_bus/worker.rb +3 -2
- data/queue-bus.gemspec +19 -18
- data/spec/adapter/publish_at_spec.rb +28 -25
- data/spec/adapter/support.rb +7 -1
- data/spec/adapter_spec.rb +4 -2
- data/spec/application_spec.rb +97 -97
- data/spec/dispatch_spec.rb +48 -51
- data/spec/driver_spec.rb +60 -58
- data/spec/heartbeat_spec.rb +26 -24
- data/spec/integration_spec.rb +41 -40
- data/spec/matcher_spec.rb +104 -102
- data/spec/publish_spec.rb +46 -46
- data/spec/publisher_spec.rb +3 -1
- data/spec/rider_spec.rb +16 -14
- data/spec/spec_helper.rb +2 -2
- data/spec/subscriber_spec.rb +227 -227
- data/spec/subscription_list_spec.rb +31 -31
- data/spec/subscription_spec.rb +37 -36
- data/spec/worker_spec.rb +17 -15
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4f35ec72b2a6256350aabd9f2f6fa6591a1b625f404a3013ea73fb24876fb9e3
|
4
|
+
data.tar.gz: 00eb73580e6160d3932f43110c182832163ab5d3f9db34d44980b831cf4f7fdc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 887ddce8443a1c17bc4db578a8d6b51d1a81906287db94ccd1f47681f30032cf485e1ed30e1a0feb56cc05b17162fae8f2ce28d07bad4b943a6dce7b40302c40
|
7
|
+
data.tar.gz: 34394be52205690b867bc2402563a9e970fa899250009481ed460f4b4909b8b8f3782bb4c5915d33eaccf6bebf2eebcedf8961628e539af6cde08646517f792f
|
@@ -0,0 +1,21 @@
|
|
1
|
+
version: 2.1
|
2
|
+
orbs:
|
3
|
+
ruby: circleci/ruby@0.1.2
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
docker:
|
8
|
+
- image: circleci/ruby:2.6.3-stretch-node
|
9
|
+
- image: circleci/redis:4.0.12-alpine
|
10
|
+
executor: ruby/default
|
11
|
+
steps:
|
12
|
+
- checkout
|
13
|
+
- run:
|
14
|
+
name: Which bundler?
|
15
|
+
command: bundle -v
|
16
|
+
- ruby/bundle-install
|
17
|
+
- run:
|
18
|
+
name: Run tests
|
19
|
+
environment:
|
20
|
+
REDIS_URL: redis://127.0.0.1:6379
|
21
|
+
command: bundle exec rspec spec
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
6
6
|
|
7
7
|
## [Unreleased]
|
8
8
|
|
9
|
+
## [0.9.1]
|
10
|
+
|
11
|
+
### Added
|
12
|
+
- Documented some of the major classes and modules
|
13
|
+
|
14
|
+
### Fixed
|
15
|
+
- Ran the rubocop autocorrect on the entire codebase.
|
16
|
+
- Fixed issue that prevented heartbeat events from firing under certain conditions
|
17
|
+
|
9
18
|
## [0.9.0]
|
10
19
|
|
11
20
|
### Added
|
data/Gemfile
CHANGED
data/Rakefile
CHANGED
data/lib/queue-bus.rb
CHANGED
@@ -1,8 +1,11 @@
|
|
1
|
-
|
2
|
-
require "forwardable"
|
1
|
+
# frozen_string_literal: true
|
3
2
|
|
4
|
-
|
3
|
+
require 'queue_bus/version'
|
4
|
+
require 'forwardable'
|
5
5
|
|
6
|
+
# The main QueueBus module. Most operations you will need to execute should be executed
|
7
|
+
# on this top level domain.
|
8
|
+
module QueueBus
|
6
9
|
autoload :Application, 'queue_bus/application'
|
7
10
|
autoload :Config, 'queue_bus/config'
|
8
11
|
autoload :Dispatch, 'queue_bus/dispatch'
|
@@ -22,25 +25,25 @@ module QueueBus
|
|
22
25
|
autoload :Util, 'queue_bus/util'
|
23
26
|
autoload :Worker, 'queue_bus/worker'
|
24
27
|
|
28
|
+
# A module for all adapters, current and future.
|
25
29
|
module Adapters
|
26
30
|
autoload :Base, 'queue_bus/adapters/base'
|
27
31
|
autoload :Data, 'queue_bus/adapters/data'
|
28
32
|
end
|
29
33
|
|
30
34
|
class << self
|
31
|
-
|
32
35
|
include Publishing
|
33
36
|
extend Forwardable
|
34
37
|
|
35
38
|
def_delegators :config, :default_app_key=, :default_app_key,
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
39
|
+
:default_queue=, :default_queue,
|
40
|
+
:local_mode=, :local_mode, :with_local_mode,
|
41
|
+
:before_publish=, :before_publish_callback,
|
42
|
+
:logger=, :logger, :log_application, :log_worker,
|
43
|
+
:hostname=, :hostname,
|
44
|
+
:adapter=, :adapter, :has_adapter?,
|
45
|
+
:incoming_queue=, :incoming_queue,
|
46
|
+
:redis, :worker_middleware_stack
|
44
47
|
|
45
48
|
def_delegators :_dispatchers, :dispatch, :dispatchers, :dispatcher_by_key, :dispatcher_execute
|
46
49
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
4
|
module Adapters
|
3
5
|
class Base
|
@@ -19,12 +21,12 @@ module QueueBus
|
|
19
21
|
raise NotImplementedError
|
20
22
|
end
|
21
23
|
|
22
|
-
def enqueue(
|
24
|
+
def enqueue(_queue_name, _klass, _json)
|
23
25
|
# enqueue the given class (Driver/Rider) in your queue
|
24
26
|
raise NotImplementedError
|
25
27
|
end
|
26
28
|
|
27
|
-
def enqueue_at(
|
29
|
+
def enqueue_at(_epoch_seconds, _queue_name, _klass, _json)
|
28
30
|
# enqueue the given class (Publisher) in your queue to run at given time
|
29
31
|
raise NotImplementedError
|
30
32
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# a base adapter just for publishing and redis connection
|
2
4
|
module QueueBus
|
3
5
|
module Adapters
|
@@ -6,17 +8,16 @@ module QueueBus
|
|
6
8
|
# nothing to do
|
7
9
|
end
|
8
10
|
|
9
|
-
|
10
|
-
@redis = client
|
11
|
-
end
|
11
|
+
attr_writer :redis
|
12
12
|
|
13
13
|
def redis(&block)
|
14
|
-
raise
|
14
|
+
raise 'no redis instance set' unless @redis
|
15
|
+
|
15
16
|
block.call(@redis)
|
16
17
|
end
|
17
18
|
|
18
19
|
def enqueue(queue_name, klass, json)
|
19
|
-
push(queue_name, :
|
20
|
+
push(queue_name, class: klass.to_s, args: [json])
|
20
21
|
end
|
21
22
|
|
22
23
|
def enqueue_at(epoch_seconds, queue_name, klass, json)
|
@@ -24,7 +25,7 @@ module QueueBus
|
|
24
25
|
delayed_push(epoch_seconds, item)
|
25
26
|
end
|
26
27
|
|
27
|
-
def setup_heartbeat!(
|
28
|
+
def setup_heartbeat!(_queue_name)
|
28
29
|
raise NotImplementedError
|
29
30
|
end
|
30
31
|
|
@@ -32,13 +33,13 @@ module QueueBus
|
|
32
33
|
|
33
34
|
def push(queue, item)
|
34
35
|
watch_queue(queue)
|
35
|
-
|
36
|
+
redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
|
36
37
|
end
|
37
38
|
|
38
39
|
# Used internally to keep track of which queues we've created.
|
39
40
|
# Don't call this directly.
|
40
41
|
def watch_queue(queue)
|
41
|
-
|
42
|
+
redis { |redis| redis.sadd(:queues, queue.to_s) }
|
42
43
|
end
|
43
44
|
|
44
45
|
# Used internally to stuff the item into the schedule sorted list.
|
@@ -46,7 +47,7 @@ module QueueBus
|
|
46
47
|
# Insertion if O(log(n)).
|
47
48
|
# Returns true if it's the first job to be scheduled at that time, else false
|
48
49
|
def delayed_push(timestamp, item)
|
49
|
-
|
50
|
+
redis do |redis|
|
50
51
|
# First add this item to the list for this timestamp
|
51
52
|
redis.rpush("delayed:#{timestamp.to_i}", ::QueueBus::Util.encode(item))
|
52
53
|
|
@@ -56,9 +57,9 @@ module QueueBus
|
|
56
57
|
redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
|
57
58
|
end
|
58
59
|
end
|
59
|
-
|
60
|
+
|
60
61
|
def delayed_job_to_hash_with_queue(queue, klass, args)
|
61
|
-
{:
|
62
|
+
{ class: klass.to_s, args: args, queue: queue }
|
62
63
|
end
|
63
64
|
end
|
64
65
|
end
|
@@ -1,33 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# An Application is the top level unifier for a number of subscriptions. It allows for
|
5
|
+
# the toggling of an entire applications subscriptions.
|
2
6
|
class Application
|
3
|
-
|
4
7
|
class << self
|
5
|
-
|
6
8
|
def all
|
7
9
|
# note the names arent the same as we started with
|
8
|
-
::QueueBus.redis { |redis| redis.smembers(app_list_key).collect{ |val| new(val) } }
|
10
|
+
::QueueBus.redis { |redis| redis.smembers(app_list_key).collect { |val| new(val) } }
|
9
11
|
end
|
10
12
|
end
|
11
13
|
|
12
14
|
attr_reader :app_key, :redis_key
|
13
15
|
|
14
|
-
|
15
16
|
def initialize(app_key)
|
16
17
|
@app_key = self.class.normalize(app_key)
|
17
18
|
@redis_key = "#{self.class.app_single_key}:#{@app_key}"
|
18
19
|
# raise error if only other chars
|
19
|
-
raise
|
20
|
+
raise 'Invalid application name' if @app_key.gsub('_', '').empty?
|
20
21
|
end
|
21
22
|
|
22
23
|
def subscribe(subscription_list, log = false)
|
23
24
|
@subscriptions = nil
|
24
25
|
|
25
|
-
if subscription_list
|
26
|
+
if subscription_list.nil? || subscription_list.empty?
|
26
27
|
unsubscribe
|
27
28
|
return true
|
28
29
|
end
|
29
30
|
|
30
|
-
temp_key = "temp_#{redis_key}:#{rand(
|
31
|
+
temp_key = "temp_#{redis_key}:#{rand(999_999_999)}"
|
31
32
|
|
32
33
|
::QueueBus.redis do |redis|
|
33
34
|
redis_hash = subscription_list.to_redis
|
@@ -39,9 +40,7 @@ module QueueBus
|
|
39
40
|
redis.rename(temp_key, redis_key)
|
40
41
|
redis.sadd(self.class.app_list_key, app_key)
|
41
42
|
|
42
|
-
if log
|
43
|
-
redis.hgetall(redis_key).inspect
|
44
|
-
end
|
43
|
+
redis.hgetall(redis_key).inspect if log
|
45
44
|
end
|
46
45
|
|
47
46
|
true
|
@@ -68,7 +67,7 @@ module QueueBus
|
|
68
67
|
def subscription_matches(attributes)
|
69
68
|
out = subscriptions.matches(attributes)
|
70
69
|
out.each do |sub|
|
71
|
-
sub.app_key =
|
70
|
+
sub.app_key = app_key
|
72
71
|
end
|
73
72
|
out
|
74
73
|
end
|
@@ -84,15 +83,15 @@ module QueueBus
|
|
84
83
|
protected
|
85
84
|
|
86
85
|
def self.normalize(val)
|
87
|
-
val.to_s.gsub(/\W/,
|
86
|
+
val.to_s.gsub(/\W/, '_').downcase
|
88
87
|
end
|
89
88
|
|
90
89
|
def self.app_list_key
|
91
|
-
|
90
|
+
'bus_apps'
|
92
91
|
end
|
93
92
|
|
94
93
|
def self.app_single_key
|
95
|
-
|
94
|
+
'bus_app'
|
96
95
|
end
|
97
96
|
|
98
97
|
def event_queues
|
@@ -116,6 +115,5 @@ module QueueBus
|
|
116
115
|
end
|
117
116
|
out
|
118
117
|
end
|
119
|
-
|
120
118
|
end
|
121
119
|
end
|
data/lib/queue_bus/dispatch.rb
CHANGED
@@ -1,23 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Creates a DSL for apps to define their blocks to run for event_types
|
2
4
|
|
3
5
|
module QueueBus
|
6
|
+
# A Dispatch object can be used to declare an application along with it's various subscriptions.
|
4
7
|
class Dispatch
|
5
|
-
|
6
8
|
attr_reader :app_key, :subscriptions
|
7
|
-
|
9
|
+
|
8
10
|
def initialize(app_key)
|
9
11
|
@app_key = Application.normalize(app_key)
|
10
12
|
@subscriptions = SubscriptionList.new
|
11
13
|
end
|
12
|
-
|
14
|
+
|
13
15
|
def size
|
14
16
|
@subscriptions.size
|
15
17
|
end
|
16
|
-
|
18
|
+
|
17
19
|
def subscribe(key, matcher_hash = nil, &block)
|
18
|
-
dispatch_event(
|
20
|
+
dispatch_event('default', key, matcher_hash, block)
|
19
21
|
end
|
20
|
-
|
22
|
+
|
21
23
|
# allows definitions of other queues
|
22
24
|
def method_missing(method_name, *args, &block)
|
23
25
|
if args.size == 1 && block
|
@@ -28,7 +30,7 @@ module QueueBus
|
|
28
30
|
super
|
29
31
|
end
|
30
32
|
end
|
31
|
-
|
33
|
+
|
32
34
|
def execute(key, attributes)
|
33
35
|
sub = subscriptions.key(key)
|
34
36
|
if sub
|
@@ -41,17 +43,17 @@ module QueueBus
|
|
41
43
|
def subscription_matches(attributes)
|
42
44
|
out = subscriptions.matches(attributes)
|
43
45
|
out.each do |sub|
|
44
|
-
sub.app_key =
|
46
|
+
sub.app_key = app_key
|
45
47
|
end
|
46
48
|
out
|
47
49
|
end
|
48
|
-
|
50
|
+
|
49
51
|
def dispatch_event(queue, key, matcher_hash, block)
|
50
52
|
# if not matcher_hash, assume key is a event_type regex
|
51
|
-
matcher_hash ||= {
|
52
|
-
add_subscription("#{app_key}_#{queue}", key,
|
53
|
+
matcher_hash ||= { 'bus_event_type' => key }
|
54
|
+
add_subscription("#{app_key}_#{queue}", key, '::QueueBus::Rider', matcher_hash, block)
|
53
55
|
end
|
54
|
-
|
56
|
+
|
55
57
|
def add_subscription(queue_name, key, class_name, matcher_hash = nil, block)
|
56
58
|
sub = Subscription.register(queue_name, key, class_name, matcher_hash, block)
|
57
59
|
subscriptions.add(sub)
|
@@ -1,26 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
4
|
+
# A collection of Dispatches
|
5
|
+
#
|
6
|
+
# Each Dispatch is an application with it's own set of subscriptions. This is a master object
|
7
|
+
# that provides some basic controls over the set of applications.
|
2
8
|
class Dispatchers
|
3
|
-
|
9
|
+
# Fetches a dispatch for the application key and binds the provided block to it.
|
10
|
+
def dispatch(app_key = nil, &block)
|
4
11
|
dispatcher = dispatcher_by_key(app_key)
|
5
12
|
dispatcher.instance_eval(&block)
|
6
13
|
dispatcher
|
7
14
|
end
|
8
|
-
|
15
|
+
|
9
16
|
def dispatchers
|
10
17
|
@dispatchers ||= {}
|
11
18
|
@dispatchers.values
|
12
19
|
end
|
13
|
-
|
20
|
+
|
14
21
|
def dispatcher_by_key(app_key)
|
15
22
|
app_key = Application.normalize(app_key || ::QueueBus.default_app_key)
|
16
23
|
@dispatchers ||= {}
|
17
24
|
@dispatchers[app_key] ||= Dispatch.new(app_key)
|
18
25
|
end
|
19
|
-
|
26
|
+
|
20
27
|
def dispatcher_execute(app_key, key, attributes)
|
21
28
|
@dispatchers ||= {}
|
22
29
|
dispatcher = @dispatchers[app_key]
|
23
|
-
dispatcher
|
30
|
+
dispatcher&.execute(key, attributes)
|
24
31
|
end
|
25
32
|
end
|
26
33
|
end
|
data/lib/queue_bus/driver.rb
CHANGED
@@ -1,7 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QueueBus
|
2
|
-
#
|
4
|
+
# Fans out an event to multiple queues
|
5
|
+
#
|
6
|
+
# When a single event is broadcast, it may have zero to many subscriptions attached to it.
|
7
|
+
# The Driver is what is run in order to look up the subscription matches and enqueue each
|
8
|
+
# of the jobs. It uses the class_name supplied by the subscription to know which class will
|
9
|
+
# be performed.
|
3
10
|
class Driver
|
4
|
-
|
5
11
|
class << self
|
6
12
|
def subscription_matches(attributes)
|
7
13
|
out = []
|
@@ -12,24 +18,23 @@ module QueueBus
|
|
12
18
|
out
|
13
19
|
end
|
14
20
|
|
15
|
-
def perform(attributes={})
|
16
|
-
raise
|
21
|
+
def perform(attributes = {})
|
22
|
+
raise 'No attributes passed' if attributes.empty?
|
17
23
|
|
18
24
|
::QueueBus.log_worker("Driver running: #{attributes.inspect}")
|
19
25
|
|
20
26
|
subscription_matches(attributes).each do |sub|
|
21
27
|
::QueueBus.log_worker(" ...sending to #{sub.queue_name} queue with class #{sub.class_name} for app #{sub.app_key} because of subscription: #{sub.key}")
|
22
28
|
|
23
|
-
bus_attr = {
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
29
|
+
bus_attr = { 'bus_driven_at' => Time.now.to_i,
|
30
|
+
'bus_rider_queue' => sub.queue_name,
|
31
|
+
'bus_rider_app_key' => sub.app_key,
|
32
|
+
'bus_rider_sub_key' => sub.key,
|
33
|
+
'bus_rider_class_name' => sub.class_name }
|
28
34
|
bus_attr = bus_attr.merge(attributes || {})
|
29
35
|
::QueueBus.enqueue_to(sub.queue_name, sub.class_name, bus_attr)
|
30
36
|
end
|
31
37
|
end
|
32
38
|
end
|
33
|
-
|
34
39
|
end
|
35
40
|
end
|