queue-bus 0.6.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +21 -0
  3. data/.rubocop.yml +35 -0
  4. data/CHANGELOG.md +38 -0
  5. data/Gemfile +4 -2
  6. data/README.mdown +16 -0
  7. data/Rakefile +2 -0
  8. data/lib/queue-bus.rb +15 -13
  9. data/lib/queue_bus/adapters/base.rb +4 -2
  10. data/lib/queue_bus/adapters/data.rb +12 -11
  11. data/lib/queue_bus/application.rb +13 -15
  12. data/lib/queue_bus/config.rb +64 -66
  13. data/lib/queue_bus/dispatch.rb +14 -12
  14. data/lib/queue_bus/dispatchers.rb +12 -5
  15. data/lib/queue_bus/driver.rb +15 -10
  16. data/lib/queue_bus/heartbeat.rb +32 -30
  17. data/lib/queue_bus/local.rb +9 -9
  18. data/lib/queue_bus/matcher.rb +36 -27
  19. data/lib/queue_bus/publisher.rb +7 -5
  20. data/lib/queue_bus/publishing.rb +31 -24
  21. data/lib/queue_bus/rider.rb +26 -22
  22. data/lib/queue_bus/subscriber.rb +20 -14
  23. data/lib/queue_bus/subscription.rb +25 -15
  24. data/lib/queue_bus/subscription_list.rb +30 -12
  25. data/lib/queue_bus/task_manager.rb +18 -16
  26. data/lib/queue_bus/tasks.rb +27 -11
  27. data/lib/queue_bus/util.rb +11 -8
  28. data/lib/queue_bus/version.rb +3 -1
  29. data/lib/queue_bus/worker.rb +3 -2
  30. data/queue-bus.gemspec +19 -17
  31. data/spec/adapter/publish_at_spec.rb +28 -25
  32. data/spec/adapter/support.rb +7 -1
  33. data/spec/adapter_spec.rb +4 -2
  34. data/spec/application_spec.rb +97 -97
  35. data/spec/config_spec.rb +116 -40
  36. data/spec/dispatch_spec.rb +48 -51
  37. data/spec/driver_spec.rb +60 -58
  38. data/spec/heartbeat_spec.rb +26 -24
  39. data/spec/integration_spec.rb +41 -40
  40. data/spec/matcher_spec.rb +104 -102
  41. data/spec/publish_spec.rb +46 -46
  42. data/spec/publisher_spec.rb +3 -1
  43. data/spec/rider_spec.rb +16 -14
  44. data/spec/spec_helper.rb +12 -7
  45. data/spec/subscriber_spec.rb +227 -227
  46. data/spec/subscription_list_spec.rb +31 -31
  47. data/spec/subscription_spec.rb +37 -36
  48. data/spec/worker_spec.rb +17 -15
  49. metadata +21 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f18ed6e21b956ce928e0c3ede7e1493eb7806413706e92e1f4c991667e9e06b8
4
- data.tar.gz: 4043918b7705741e3719d6460b9823367b9e271e6344193c2b616aab063a4d9a
3
+ metadata.gz: 4f35ec72b2a6256350aabd9f2f6fa6591a1b625f404a3013ea73fb24876fb9e3
4
+ data.tar.gz: 00eb73580e6160d3932f43110c182832163ab5d3f9db34d44980b831cf4f7fdc
5
5
  SHA512:
6
- metadata.gz: 6894d383941edf09df6d245eaffc57ecd2ef3185bb3196d89a29c7714cc7b3f849639e285764ee57832158a566538bc2c1b8b6691af2d73ab5a3760d8893ef5e
7
- data.tar.gz: 9b08d34ff3309f039640a1c154b5f33cfc186f69f18511a09201e4eeb5553ef8f5b8ad053405fc6f9bf3658d5750543d7fa92d431d88ba72af5c3ab8f0d50e45
6
+ metadata.gz: 887ddce8443a1c17bc4db578a8d6b51d1a81906287db94ccd1f47681f30032cf485e1ed30e1a0feb56cc05b17162fae8f2ce28d07bad4b943a6dce7b40302c40
7
+ data.tar.gz: 34394be52205690b867bc2402563a9e970fa899250009481ed460f4b4909b8b8f3782bb4c5915d33eaccf6bebf2eebcedf8961628e539af6cde08646517f792f
@@ -0,0 +1,21 @@
1
+ version: 2.1
2
+ orbs:
3
+ ruby: circleci/ruby@0.1.2
4
+
5
+ jobs:
6
+ build:
7
+ docker:
8
+ - image: circleci/ruby:2.6.3-stretch-node
9
+ - image: circleci/redis:4.0.12-alpine
10
+ executor: ruby/default
11
+ steps:
12
+ - checkout
13
+ - run:
14
+ name: Which bundler?
15
+ command: bundle -v
16
+ - ruby/bundle-install
17
+ - run:
18
+ name: Run tests
19
+ environment:
20
+ REDIS_URL: redis://127.0.0.1:6379
21
+ command: bundle exec rspec spec
@@ -0,0 +1,35 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4
3
+ ExtraDetails: true
4
+
5
+ # http://rubocop.readthedocs.io/en/latest/cops_style/#stylefrozenstringliteralcomment
6
+ Style/FrozenStringLiteralComment:
7
+ Enabled: true
8
+
9
+ # https://rubocop.readthedocs.io/en/latest/cops_style/#styledatetime
10
+ Style/DateTime:
11
+ Enabled: true
12
+
13
+ # http://rubocop.readthedocs.io/en/latest/cops_metrics/#metricslinelength
14
+ Metrics/LineLength:
15
+ Max: 100
16
+
17
+ Metrics/MethodLength:
18
+ Max: 15
19
+
20
+ Metrics/BlockLength:
21
+ Exclude:
22
+ - spec/**/*
23
+
24
+ # https://rubocop.readthedocs.io/en/latest/cops_layout/#layoutdotposition
25
+ Layout/DotPosition:
26
+ Enabled: true
27
+ EnforcedStyle: leading
28
+
29
+ # https://rubocop.readthedocs.io/en/latest/cops_style/#stylehashsyntax
30
+ Style/HashSyntax:
31
+ Enabled: true
32
+
33
+ # https://rubocop-rspec.readthedocs.io/en/latest/cops_rspec/#rspecfocus
34
+ RSpec/Focus:
35
+ Enabled: true
@@ -6,6 +6,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.9.1]
10
+
11
+ ### Added
12
+ - Documented some of the major classes and modules
13
+
14
+ ### Fixed
15
+ - Ran the rubocop autocorrect on the entire codebase.
16
+ - Fixed issue that prevented heartbeat events from firing under certain conditions
17
+
18
+ ## [0.9.0]
19
+
20
+ ### Added
21
+ - Adds rake tasks to list scheduled jobs as csv
22
+
23
+ ## [0.8.1]
24
+
25
+ ### Fixed
26
+ - `with_local_mode` breaks subsequent calls to `local_mode` on versions less than 2.6.
27
+
28
+ ## [0.8.0]
29
+
30
+ ### Added
31
+ - Adds `QueueBus.with_local_mode` method. Useful when working with a multithreaded environment.
32
+
33
+ ## [0.7.0]
34
+
35
+ ### Added
36
+ - Adds `QueueBus.has_adapter?` to check whether the adapter is set.
37
+
38
+ ### Changed
39
+ - Now uses `Process.hostname` to determine hostname versus relying on unix shell.
40
+ - Rubocop is now a dev dependency.
41
+ - Accessors to config are now done with actual attrs.
42
+ - Logging with the adapter will use the logger if present.
43
+
44
+ ### Fixed
45
+ - Passing a class to `adapter=` would error on a `NameError`.
46
+
9
47
  ## [0.6.0]
10
48
 
11
49
  ### Added
data/Gemfile CHANGED
@@ -1,5 +1,7 @@
1
- source "http://rubygems.org"
1
+ # frozen_string_literal: true
2
+
3
+ source 'http://rubygems.org'
2
4
 
3
5
  gemspec
4
6
 
5
- gem "rake"
7
+ gem 'rake'
@@ -141,6 +141,22 @@ event to the appropriate code block.
141
141
  You can also say `QueueBus.local_mode = :suppress` to turn off publishing altogether.
142
142
  This can be helpful inside some sort of migration, for example.
143
143
 
144
+ #### Thread Safe Local Modes
145
+
146
+ **!! This is important if you are using workers that utilize multiple threads, such as Sidekiq !!**
147
+
148
+ The above setting is global to the ruby process and modifying it will impact all threads that are
149
+ currently using QueueBus. If you want to isolate a thread or block of code from QueueBus, you can
150
+ use the method `with_local_mode`:
151
+
152
+ ```ruby
153
+ QueueBus.with_local_mode(:suppress) do
154
+ # QueueBus will be suppressed on this thread, within this block.
155
+ end
156
+ ```
157
+
158
+ The previous value of `local_mode` will be restored after the block exits.
159
+
144
160
  ### TODO
145
161
 
146
162
  * Replace local modes with adapters
data/Rakefile CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  $LOAD_PATH.unshift 'lib'
@@ -1,8 +1,11 @@
1
- require "queue_bus/version"
2
- require "forwardable"
1
+ # frozen_string_literal: true
3
2
 
4
- module QueueBus
3
+ require 'queue_bus/version'
4
+ require 'forwardable'
5
5
 
6
+ # The main QueueBus module. Most operations you will need to execute should be executed
7
+ # on this top level domain.
8
+ module QueueBus
6
9
  autoload :Application, 'queue_bus/application'
7
10
  autoload :Config, 'queue_bus/config'
8
11
  autoload :Dispatch, 'queue_bus/dispatch'
@@ -22,25 +25,25 @@ module QueueBus
22
25
  autoload :Util, 'queue_bus/util'
23
26
  autoload :Worker, 'queue_bus/worker'
24
27
 
28
+ # A module for all adapters, current and future.
25
29
  module Adapters
26
30
  autoload :Base, 'queue_bus/adapters/base'
27
31
  autoload :Data, 'queue_bus/adapters/data'
28
32
  end
29
33
 
30
34
  class << self
31
-
32
35
  include Publishing
33
36
  extend Forwardable
34
37
 
35
38
  def_delegators :config, :default_app_key=, :default_app_key,
36
- :default_queue=, :default_queue,
37
- :local_mode=, :local_mode,
38
- :before_publish=, :before_publish_callback,
39
- :logger=, :logger, :log_application, :log_worker,
40
- :hostname=, :hostname,
41
- :adapter=, :adapter,
42
- :incoming_queue=, :incoming_queue,
43
- :redis, :worker_middleware_stack
39
+ :default_queue=, :default_queue,
40
+ :local_mode=, :local_mode, :with_local_mode,
41
+ :before_publish=, :before_publish_callback,
42
+ :logger=, :logger, :log_application, :log_worker,
43
+ :hostname=, :hostname,
44
+ :adapter=, :adapter, :has_adapter?,
45
+ :incoming_queue=, :incoming_queue,
46
+ :redis, :worker_middleware_stack
44
47
 
45
48
  def_delegators :_dispatchers, :dispatch, :dispatchers, :dispatcher_by_key, :dispatcher_execute
46
49
 
@@ -60,5 +63,4 @@ module QueueBus
60
63
  @_dispatchers ||= ::QueueBus::Dispatchers.new
61
64
  end
62
65
  end
63
-
64
66
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
4
  module Adapters
3
5
  class Base
@@ -19,12 +21,12 @@ module QueueBus
19
21
  raise NotImplementedError
20
22
  end
21
23
 
22
- def enqueue(queue_name, klass, json)
24
+ def enqueue(_queue_name, _klass, _json)
23
25
  # enqueue the given class (Driver/Rider) in your queue
24
26
  raise NotImplementedError
25
27
  end
26
28
 
27
- def enqueue_at(epoch_seconds, queue_name, klass, json)
29
+ def enqueue_at(_epoch_seconds, _queue_name, _klass, _json)
28
30
  # enqueue the given class (Publisher) in your queue to run at given time
29
31
  raise NotImplementedError
30
32
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # a base adapter just for publishing and redis connection
2
4
  module QueueBus
3
5
  module Adapters
@@ -6,17 +8,16 @@ module QueueBus
6
8
  # nothing to do
7
9
  end
8
10
 
9
- def redis=(client)
10
- @redis = client
11
- end
11
+ attr_writer :redis
12
12
 
13
13
  def redis(&block)
14
- raise "no redis instance set" unless @redis
14
+ raise 'no redis instance set' unless @redis
15
+
15
16
  block.call(@redis)
16
17
  end
17
18
 
18
19
  def enqueue(queue_name, klass, json)
19
- push(queue_name, :class => klass.to_s, :args => [json])
20
+ push(queue_name, class: klass.to_s, args: [json])
20
21
  end
21
22
 
22
23
  def enqueue_at(epoch_seconds, queue_name, klass, json)
@@ -24,7 +25,7 @@ module QueueBus
24
25
  delayed_push(epoch_seconds, item)
25
26
  end
26
27
 
27
- def setup_heartbeat!(queue_name)
28
+ def setup_heartbeat!(_queue_name)
28
29
  raise NotImplementedError
29
30
  end
30
31
 
@@ -32,13 +33,13 @@ module QueueBus
32
33
 
33
34
  def push(queue, item)
34
35
  watch_queue(queue)
35
- self.redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
36
+ redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
36
37
  end
37
38
 
38
39
  # Used internally to keep track of which queues we've created.
39
40
  # Don't call this directly.
40
41
  def watch_queue(queue)
41
- self.redis { |redis| redis.sadd(:queues, queue.to_s) }
42
+ redis { |redis| redis.sadd(:queues, queue.to_s) }
42
43
  end
43
44
 
44
45
  # Used internally to stuff the item into the schedule sorted list.
@@ -46,7 +47,7 @@ module QueueBus
46
47
  # Insertion if O(log(n)).
47
48
  # Returns true if it's the first job to be scheduled at that time, else false
48
49
  def delayed_push(timestamp, item)
49
- self.redis do |redis|
50
+ redis do |redis|
50
51
  # First add this item to the list for this timestamp
51
52
  redis.rpush("delayed:#{timestamp.to_i}", ::QueueBus::Util.encode(item))
52
53
 
@@ -56,9 +57,9 @@ module QueueBus
56
57
  redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
57
58
  end
58
59
  end
59
-
60
+
60
61
  def delayed_job_to_hash_with_queue(queue, klass, args)
61
- {:class => klass.to_s, :args => args, :queue => queue}
62
+ { class: klass.to_s, args: args, queue: queue }
62
63
  end
63
64
  end
64
65
  end
@@ -1,33 +1,34 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # An Application is the top level unifier for a number of subscriptions. It allows for
5
+ # the toggling of an entire applications subscriptions.
2
6
  class Application
3
-
4
7
  class << self
5
-
6
8
  def all
7
9
  # note the names arent the same as we started with
8
- ::QueueBus.redis { |redis| redis.smembers(app_list_key).collect{ |val| new(val) } }
10
+ ::QueueBus.redis { |redis| redis.smembers(app_list_key).collect { |val| new(val) } }
9
11
  end
10
12
  end
11
13
 
12
14
  attr_reader :app_key, :redis_key
13
15
 
14
-
15
16
  def initialize(app_key)
16
17
  @app_key = self.class.normalize(app_key)
17
18
  @redis_key = "#{self.class.app_single_key}:#{@app_key}"
18
19
  # raise error if only other chars
19
- raise "Invalid application name" if @app_key.gsub("_", "").size == 0
20
+ raise 'Invalid application name' if @app_key.gsub('_', '').empty?
20
21
  end
21
22
 
22
23
  def subscribe(subscription_list, log = false)
23
24
  @subscriptions = nil
24
25
 
25
- if subscription_list == nil || subscription_list.size == 0
26
+ if subscription_list.nil? || subscription_list.empty?
26
27
  unsubscribe
27
28
  return true
28
29
  end
29
30
 
30
- temp_key = "temp_#{redis_key}:#{rand(999999999)}"
31
+ temp_key = "temp_#{redis_key}:#{rand(999_999_999)}"
31
32
 
32
33
  ::QueueBus.redis do |redis|
33
34
  redis_hash = subscription_list.to_redis
@@ -39,9 +40,7 @@ module QueueBus
39
40
  redis.rename(temp_key, redis_key)
40
41
  redis.sadd(self.class.app_list_key, app_key)
41
42
 
42
- if log
43
- redis.hgetall(redis_key).inspect
44
- end
43
+ redis.hgetall(redis_key).inspect if log
45
44
  end
46
45
 
47
46
  true
@@ -68,7 +67,7 @@ module QueueBus
68
67
  def subscription_matches(attributes)
69
68
  out = subscriptions.matches(attributes)
70
69
  out.each do |sub|
71
- sub.app_key = self.app_key
70
+ sub.app_key = app_key
72
71
  end
73
72
  out
74
73
  end
@@ -84,15 +83,15 @@ module QueueBus
84
83
  protected
85
84
 
86
85
  def self.normalize(val)
87
- val.to_s.gsub(/\W/, "_").downcase
86
+ val.to_s.gsub(/\W/, '_').downcase
88
87
  end
89
88
 
90
89
  def self.app_list_key
91
- "bus_apps"
90
+ 'bus_apps'
92
91
  end
93
92
 
94
93
  def self.app_single_key
95
- "bus_app"
94
+ 'bus_app'
96
95
  end
97
96
 
98
97
  def event_queues
@@ -116,6 +115,5 @@ module QueueBus
116
115
  end
117
116
  out
118
117
  end
119
-
120
118
  end
121
119
  end
@@ -1,102 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'socket'
4
+ require 'logger'
5
+
1
6
  module QueueBus
7
+ # This class contains all the configuration for a running queue bus application.
2
8
  class Config
3
- def adapter=val
4
- raise "Adapter already set to #{@adapter_instance.class.name}" if @adapter_instance
5
- if val.is_a?(Class)
6
- @adapter_instance = name_or_klass.new
7
- elsif val.is_a?(::QueueBus::Adapters::Base)
8
- @adapter_instance = val
9
- else
10
- class_name = ::QueueBus::Util.classify(val)
11
- @adapter_instance = ::QueueBus::Util.constantize("::QueueBus::Adapters::#{class_name}").new
12
- end
13
- @adapter_instance
14
- end
9
+ attr_accessor :default_queue, :hostname, :incoming_queue, :logger
15
10
 
16
- def adapter
17
- return @adapter_instance if @adapter_instance
18
- raise "no adapter has been set"
19
- end
11
+ attr_reader :worker_middleware_stack
12
+ attr_writer :local_mode
20
13
 
21
- def redis(&block)
22
- # TODO: could allow setting for non-redis adapters
23
- adapter.redis(&block)
14
+ def initialize
15
+ @worker_middleware_stack = QueueBus::Middleware::Stack.new
16
+ @incoming_queue = 'bus_incoming'
17
+ @hostname = Socket.gethostname
24
18
  end
25
19
 
26
- def default_app_key=val
27
- @default_app_key = Application.normalize(val)
28
- end
20
+ # A wrapper that is always "truthy" but can contain an inner value. This is useful for
21
+ # checking that a thread local variable is set to a value, even if that value happens to
22
+ # be nil. This is important because setting a thread local value to nil will cause it to
23
+ # be deleted.
24
+ Wrap = Struct.new(:value)
29
25
 
30
- def default_app_key
31
- @default_app_key
32
- end
26
+ LOCAL_MODE_VAR = :queue_bus_local_mode
33
27
 
34
- def default_queue=val
35
- @default_queue = val
28
+ # Returns the current local mode of QueueBus
29
+ def local_mode
30
+ if Thread.current.thread_variable_get(LOCAL_MODE_VAR).is_a?(Wrap)
31
+ Thread.current.thread_variable_get(LOCAL_MODE_VAR).value
32
+ else
33
+ @local_mode
34
+ end
36
35
  end
37
36
 
38
- def default_queue
39
- @default_queue
37
+ # Overrides the current local mode for the duration of a block. This is a threadsafe
38
+ # implementation. After, the global setting will be resumed.
39
+ #
40
+ # @param mode [Symbol] the mode to switch to
41
+ def with_local_mode(mode)
42
+ previous = Thread.current.thread_variable_get(LOCAL_MODE_VAR)
43
+ Thread.current.thread_variable_set(LOCAL_MODE_VAR, Wrap.new(mode))
44
+ yield if block_given?
45
+ ensure
46
+ Thread.current.thread_variable_set(LOCAL_MODE_VAR, previous)
40
47
  end
41
48
 
42
- def local_mode=value
43
- @local_mode = value
44
- end
49
+ def adapter=(val)
50
+ raise "Adapter already set to #{@adapter_instance.class.name}" if has_adapter?
45
51
 
46
- def local_mode
47
- @local_mode
52
+ @adapter_instance =
53
+ if val.is_a?(Class)
54
+ val.new
55
+ elsif val.is_a?(::QueueBus::Adapters::Base)
56
+ val
57
+ else
58
+ class_name = ::QueueBus::Util.classify(val)
59
+ ::QueueBus::Util.constantize("::QueueBus::Adapters::#{class_name}").new
60
+ end
48
61
  end
49
62
 
50
- def incoming_queue=val
51
- @incoming_queue = val
52
- end
63
+ def adapter
64
+ return @adapter_instance if has_adapter?
53
65
 
54
- def incoming_queue
55
- @incoming_queue ||= "bus_incoming"
66
+ raise 'no adapter has been set'
56
67
  end
57
68
 
58
- def worker_middleware_stack
59
- @worker_middleware_stack ||= QueueBus::Middleware::Stack.new
69
+ # Checks whether an adapter is set and returns true if it is.
70
+ def has_adapter? # rubocop:disable Naming/PredicateName
71
+ !@adapter_instance.nil?
60
72
  end
61
73
 
62
- def hostname
63
- @hostname ||= `hostname 2>&1`.strip.sub(/.local/,'')
74
+ def redis(&block)
75
+ # TODO: could allow setting for non-redis adapters
76
+ adapter.redis(&block)
64
77
  end
65
78
 
66
- def hostname=val
67
- @hostname = val
79
+ attr_reader :default_app_key
80
+ def default_app_key=(val)
81
+ @default_app_key = Application.normalize(val)
68
82
  end
69
83
 
70
- def before_publish=(proc)
71
- @before_publish_callback = proc
84
+ def before_publish=(callback)
85
+ @before_publish_callback = callback
72
86
  end
73
87
 
74
88
  def before_publish_callback(attributes)
75
- if @before_publish_callback
76
- @before_publish_callback.call(attributes)
77
- end
78
- end
79
-
80
- def logger
81
- @logger
82
- end
83
-
84
- def logger=val
85
- @logger = val
89
+ @before_publish_callback&.call(attributes)
86
90
  end
87
91
 
88
92
  def log_application(message)
89
- if logger
90
- time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
91
- logger.info("** [#{time}] #$$: QueueBus #{message}")
92
- end
93
+ logger&.info(message)
93
94
  end
94
95
 
95
96
  def log_worker(message)
96
- if ENV['LOGGING'] || ENV['VERBOSE'] || ENV['VVERBOSE']
97
- time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
98
- puts "** [#{time}] #$$: #{message}"
99
- end
97
+ logger&.debug(message)
100
98
  end
101
99
  end
102
100
  end