queue-bus 0.6.0 → 0.9.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +21 -0
  3. data/.rubocop.yml +35 -0
  4. data/CHANGELOG.md +38 -0
  5. data/Gemfile +4 -2
  6. data/README.mdown +16 -0
  7. data/Rakefile +2 -0
  8. data/lib/queue-bus.rb +15 -13
  9. data/lib/queue_bus/adapters/base.rb +4 -2
  10. data/lib/queue_bus/adapters/data.rb +12 -11
  11. data/lib/queue_bus/application.rb +13 -15
  12. data/lib/queue_bus/config.rb +64 -66
  13. data/lib/queue_bus/dispatch.rb +14 -12
  14. data/lib/queue_bus/dispatchers.rb +12 -5
  15. data/lib/queue_bus/driver.rb +15 -10
  16. data/lib/queue_bus/heartbeat.rb +32 -30
  17. data/lib/queue_bus/local.rb +9 -9
  18. data/lib/queue_bus/matcher.rb +36 -27
  19. data/lib/queue_bus/publisher.rb +7 -5
  20. data/lib/queue_bus/publishing.rb +31 -24
  21. data/lib/queue_bus/rider.rb +26 -22
  22. data/lib/queue_bus/subscriber.rb +20 -14
  23. data/lib/queue_bus/subscription.rb +25 -15
  24. data/lib/queue_bus/subscription_list.rb +30 -12
  25. data/lib/queue_bus/task_manager.rb +18 -16
  26. data/lib/queue_bus/tasks.rb +27 -11
  27. data/lib/queue_bus/util.rb +11 -8
  28. data/lib/queue_bus/version.rb +3 -1
  29. data/lib/queue_bus/worker.rb +3 -2
  30. data/queue-bus.gemspec +19 -17
  31. data/spec/adapter/publish_at_spec.rb +28 -25
  32. data/spec/adapter/support.rb +7 -1
  33. data/spec/adapter_spec.rb +4 -2
  34. data/spec/application_spec.rb +97 -97
  35. data/spec/config_spec.rb +116 -40
  36. data/spec/dispatch_spec.rb +48 -51
  37. data/spec/driver_spec.rb +60 -58
  38. data/spec/heartbeat_spec.rb +26 -24
  39. data/spec/integration_spec.rb +41 -40
  40. data/spec/matcher_spec.rb +104 -102
  41. data/spec/publish_spec.rb +46 -46
  42. data/spec/publisher_spec.rb +3 -1
  43. data/spec/rider_spec.rb +16 -14
  44. data/spec/spec_helper.rb +12 -7
  45. data/spec/subscriber_spec.rb +227 -227
  46. data/spec/subscription_list_spec.rb +31 -31
  47. data/spec/subscription_spec.rb +37 -36
  48. data/spec/worker_spec.rb +17 -15
  49. metadata +21 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f18ed6e21b956ce928e0c3ede7e1493eb7806413706e92e1f4c991667e9e06b8
4
- data.tar.gz: 4043918b7705741e3719d6460b9823367b9e271e6344193c2b616aab063a4d9a
3
+ metadata.gz: 4f35ec72b2a6256350aabd9f2f6fa6591a1b625f404a3013ea73fb24876fb9e3
4
+ data.tar.gz: 00eb73580e6160d3932f43110c182832163ab5d3f9db34d44980b831cf4f7fdc
5
5
  SHA512:
6
- metadata.gz: 6894d383941edf09df6d245eaffc57ecd2ef3185bb3196d89a29c7714cc7b3f849639e285764ee57832158a566538bc2c1b8b6691af2d73ab5a3760d8893ef5e
7
- data.tar.gz: 9b08d34ff3309f039640a1c154b5f33cfc186f69f18511a09201e4eeb5553ef8f5b8ad053405fc6f9bf3658d5750543d7fa92d431d88ba72af5c3ab8f0d50e45
6
+ metadata.gz: 887ddce8443a1c17bc4db578a8d6b51d1a81906287db94ccd1f47681f30032cf485e1ed30e1a0feb56cc05b17162fae8f2ce28d07bad4b943a6dce7b40302c40
7
+ data.tar.gz: 34394be52205690b867bc2402563a9e970fa899250009481ed460f4b4909b8b8f3782bb4c5915d33eaccf6bebf2eebcedf8961628e539af6cde08646517f792f
@@ -0,0 +1,21 @@
1
+ version: 2.1
2
+ orbs:
3
+ ruby: circleci/ruby@0.1.2
4
+
5
+ jobs:
6
+ build:
7
+ docker:
8
+ - image: circleci/ruby:2.6.3-stretch-node
9
+ - image: circleci/redis:4.0.12-alpine
10
+ executor: ruby/default
11
+ steps:
12
+ - checkout
13
+ - run:
14
+ name: Which bundler?
15
+ command: bundle -v
16
+ - ruby/bundle-install
17
+ - run:
18
+ name: Run tests
19
+ environment:
20
+ REDIS_URL: redis://127.0.0.1:6379
21
+ command: bundle exec rspec spec
@@ -0,0 +1,35 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4
3
+ ExtraDetails: true
4
+
5
+ # http://rubocop.readthedocs.io/en/latest/cops_style/#stylefrozenstringliteralcomment
6
+ Style/FrozenStringLiteralComment:
7
+ Enabled: true
8
+
9
+ # https://rubocop.readthedocs.io/en/latest/cops_style/#styledatetime
10
+ Style/DateTime:
11
+ Enabled: true
12
+
13
+ # http://rubocop.readthedocs.io/en/latest/cops_metrics/#metricslinelength
14
+ Metrics/LineLength:
15
+ Max: 100
16
+
17
+ Metrics/MethodLength:
18
+ Max: 15
19
+
20
+ Metrics/BlockLength:
21
+ Exclude:
22
+ - spec/**/*
23
+
24
+ # https://rubocop.readthedocs.io/en/latest/cops_layout/#layoutdotposition
25
+ Layout/DotPosition:
26
+ Enabled: true
27
+ EnforcedStyle: leading
28
+
29
+ # https://rubocop.readthedocs.io/en/latest/cops_style/#stylehashsyntax
30
+ Style/HashSyntax:
31
+ Enabled: true
32
+
33
+ # https://rubocop-rspec.readthedocs.io/en/latest/cops_rspec/#rspecfocus
34
+ RSpec/Focus:
35
+ Enabled: true
@@ -6,6 +6,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.9.1]
10
+
11
+ ### Added
12
+ - Documented some of the major classes and modules
13
+
14
+ ### Fixed
15
+ - Ran the rubocop autocorrect on the entire codebase.
16
+ - Fixed issue that prevented heartbeat events from firing under certain conditions
17
+
18
+ ## [0.9.0]
19
+
20
+ ### Added
21
+ - Adds rake tasks to list scheduled jobs as csv
22
+
23
+ ## [0.8.1]
24
+
25
+ ### Fixed
26
+ - `with_local_mode` breaks subsequent calls to `local_mode` on versions less than 2.6.
27
+
28
+ ## [0.8.0]
29
+
30
+ ### Added
31
+ - Adds `QueueBus.with_local_mode` method. Useful when working with a multithreaded environment.
32
+
33
+ ## [0.7.0]
34
+
35
+ ### Added
36
+ - Adds `QueueBus.has_adapter?` to check whether the adapter is set.
37
+
38
+ ### Changed
39
+ - Now uses `Process.hostname` to determine hostname versus relying on unix shell.
40
+ - Rubocop is now a dev dependency.
41
+ - Accessors to config are now done with actual attrs.
42
+ - Logging with the adapter will use the logger if present.
43
+
44
+ ### Fixed
45
+ - Passing a class to `adapter=` would error on a `NameError`.
46
+
9
47
  ## [0.6.0]
10
48
 
11
49
  ### Added
data/Gemfile CHANGED
@@ -1,5 +1,7 @@
1
- source "http://rubygems.org"
1
+ # frozen_string_literal: true
2
+
3
+ source 'http://rubygems.org'
2
4
 
3
5
  gemspec
4
6
 
5
- gem "rake"
7
+ gem 'rake'
@@ -141,6 +141,22 @@ event to the appropriate code block.
141
141
  You can also say `QueueBus.local_mode = :suppress` to turn off publishing altogether.
142
142
  This can be helpful inside some sort of migration, for example.
143
143
 
144
+ #### Thread Safe Local Modes
145
+
146
+ **!! This is important if you are using workers that utilize multiple threads, such as Sidekiq !!**
147
+
148
+ The above setting is global to the ruby process and modifying it will impact all threads that are
149
+ currently using QueueBus. If you want to isolate a thread or block of code from QueueBus, you can
150
+ use the method `with_local_mode`:
151
+
152
+ ```ruby
153
+ QueueBus.with_local_mode(:suppress) do
154
+ # QueueBus will be suppressed on this thread, within this block.
155
+ end
156
+ ```
157
+
158
+ The previous value of `local_mode` will be restored after the block exits.
159
+
144
160
  ### TODO
145
161
 
146
162
  * Replace local modes with adapters
data/Rakefile CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  $LOAD_PATH.unshift 'lib'
@@ -1,8 +1,11 @@
1
- require "queue_bus/version"
2
- require "forwardable"
1
+ # frozen_string_literal: true
3
2
 
4
- module QueueBus
3
+ require 'queue_bus/version'
4
+ require 'forwardable'
5
5
 
6
+ # The main QueueBus module. Most operations you will need to execute should be executed
7
+ # on this top level domain.
8
+ module QueueBus
6
9
  autoload :Application, 'queue_bus/application'
7
10
  autoload :Config, 'queue_bus/config'
8
11
  autoload :Dispatch, 'queue_bus/dispatch'
@@ -22,25 +25,25 @@ module QueueBus
22
25
  autoload :Util, 'queue_bus/util'
23
26
  autoload :Worker, 'queue_bus/worker'
24
27
 
28
+ # A module for all adapters, current and future.
25
29
  module Adapters
26
30
  autoload :Base, 'queue_bus/adapters/base'
27
31
  autoload :Data, 'queue_bus/adapters/data'
28
32
  end
29
33
 
30
34
  class << self
31
-
32
35
  include Publishing
33
36
  extend Forwardable
34
37
 
35
38
  def_delegators :config, :default_app_key=, :default_app_key,
36
- :default_queue=, :default_queue,
37
- :local_mode=, :local_mode,
38
- :before_publish=, :before_publish_callback,
39
- :logger=, :logger, :log_application, :log_worker,
40
- :hostname=, :hostname,
41
- :adapter=, :adapter,
42
- :incoming_queue=, :incoming_queue,
43
- :redis, :worker_middleware_stack
39
+ :default_queue=, :default_queue,
40
+ :local_mode=, :local_mode, :with_local_mode,
41
+ :before_publish=, :before_publish_callback,
42
+ :logger=, :logger, :log_application, :log_worker,
43
+ :hostname=, :hostname,
44
+ :adapter=, :adapter, :has_adapter?,
45
+ :incoming_queue=, :incoming_queue,
46
+ :redis, :worker_middleware_stack
44
47
 
45
48
  def_delegators :_dispatchers, :dispatch, :dispatchers, :dispatcher_by_key, :dispatcher_execute
46
49
 
@@ -60,5 +63,4 @@ module QueueBus
60
63
  @_dispatchers ||= ::QueueBus::Dispatchers.new
61
64
  end
62
65
  end
63
-
64
66
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
2
4
  module Adapters
3
5
  class Base
@@ -19,12 +21,12 @@ module QueueBus
19
21
  raise NotImplementedError
20
22
  end
21
23
 
22
- def enqueue(queue_name, klass, json)
24
+ def enqueue(_queue_name, _klass, _json)
23
25
  # enqueue the given class (Driver/Rider) in your queue
24
26
  raise NotImplementedError
25
27
  end
26
28
 
27
- def enqueue_at(epoch_seconds, queue_name, klass, json)
29
+ def enqueue_at(_epoch_seconds, _queue_name, _klass, _json)
28
30
  # enqueue the given class (Publisher) in your queue to run at given time
29
31
  raise NotImplementedError
30
32
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # a base adapter just for publishing and redis connection
2
4
  module QueueBus
3
5
  module Adapters
@@ -6,17 +8,16 @@ module QueueBus
6
8
  # nothing to do
7
9
  end
8
10
 
9
- def redis=(client)
10
- @redis = client
11
- end
11
+ attr_writer :redis
12
12
 
13
13
  def redis(&block)
14
- raise "no redis instance set" unless @redis
14
+ raise 'no redis instance set' unless @redis
15
+
15
16
  block.call(@redis)
16
17
  end
17
18
 
18
19
  def enqueue(queue_name, klass, json)
19
- push(queue_name, :class => klass.to_s, :args => [json])
20
+ push(queue_name, class: klass.to_s, args: [json])
20
21
  end
21
22
 
22
23
  def enqueue_at(epoch_seconds, queue_name, klass, json)
@@ -24,7 +25,7 @@ module QueueBus
24
25
  delayed_push(epoch_seconds, item)
25
26
  end
26
27
 
27
- def setup_heartbeat!(queue_name)
28
+ def setup_heartbeat!(_queue_name)
28
29
  raise NotImplementedError
29
30
  end
30
31
 
@@ -32,13 +33,13 @@ module QueueBus
32
33
 
33
34
  def push(queue, item)
34
35
  watch_queue(queue)
35
- self.redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
36
+ redis { |redis| redis.rpush "queue:#{queue}", ::QueueBus::Util.encode(item) }
36
37
  end
37
38
 
38
39
  # Used internally to keep track of which queues we've created.
39
40
  # Don't call this directly.
40
41
  def watch_queue(queue)
41
- self.redis { |redis| redis.sadd(:queues, queue.to_s) }
42
+ redis { |redis| redis.sadd(:queues, queue.to_s) }
42
43
  end
43
44
 
44
45
  # Used internally to stuff the item into the schedule sorted list.
@@ -46,7 +47,7 @@ module QueueBus
46
47
  # Insertion if O(log(n)).
47
48
  # Returns true if it's the first job to be scheduled at that time, else false
48
49
  def delayed_push(timestamp, item)
49
- self.redis do |redis|
50
+ redis do |redis|
50
51
  # First add this item to the list for this timestamp
51
52
  redis.rpush("delayed:#{timestamp.to_i}", ::QueueBus::Util.encode(item))
52
53
 
@@ -56,9 +57,9 @@ module QueueBus
56
57
  redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
57
58
  end
58
59
  end
59
-
60
+
60
61
  def delayed_job_to_hash_with_queue(queue, klass, args)
61
- {:class => klass.to_s, :args => args, :queue => queue}
62
+ { class: klass.to_s, args: args, queue: queue }
62
63
  end
63
64
  end
64
65
  end
@@ -1,33 +1,34 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QueueBus
4
+ # An Application is the top level unifier for a number of subscriptions. It allows for
5
+ # the toggling of an entire applications subscriptions.
2
6
  class Application
3
-
4
7
  class << self
5
-
6
8
  def all
7
9
  # note the names arent the same as we started with
8
- ::QueueBus.redis { |redis| redis.smembers(app_list_key).collect{ |val| new(val) } }
10
+ ::QueueBus.redis { |redis| redis.smembers(app_list_key).collect { |val| new(val) } }
9
11
  end
10
12
  end
11
13
 
12
14
  attr_reader :app_key, :redis_key
13
15
 
14
-
15
16
  def initialize(app_key)
16
17
  @app_key = self.class.normalize(app_key)
17
18
  @redis_key = "#{self.class.app_single_key}:#{@app_key}"
18
19
  # raise error if only other chars
19
- raise "Invalid application name" if @app_key.gsub("_", "").size == 0
20
+ raise 'Invalid application name' if @app_key.gsub('_', '').empty?
20
21
  end
21
22
 
22
23
  def subscribe(subscription_list, log = false)
23
24
  @subscriptions = nil
24
25
 
25
- if subscription_list == nil || subscription_list.size == 0
26
+ if subscription_list.nil? || subscription_list.empty?
26
27
  unsubscribe
27
28
  return true
28
29
  end
29
30
 
30
- temp_key = "temp_#{redis_key}:#{rand(999999999)}"
31
+ temp_key = "temp_#{redis_key}:#{rand(999_999_999)}"
31
32
 
32
33
  ::QueueBus.redis do |redis|
33
34
  redis_hash = subscription_list.to_redis
@@ -39,9 +40,7 @@ module QueueBus
39
40
  redis.rename(temp_key, redis_key)
40
41
  redis.sadd(self.class.app_list_key, app_key)
41
42
 
42
- if log
43
- redis.hgetall(redis_key).inspect
44
- end
43
+ redis.hgetall(redis_key).inspect if log
45
44
  end
46
45
 
47
46
  true
@@ -68,7 +67,7 @@ module QueueBus
68
67
  def subscription_matches(attributes)
69
68
  out = subscriptions.matches(attributes)
70
69
  out.each do |sub|
71
- sub.app_key = self.app_key
70
+ sub.app_key = app_key
72
71
  end
73
72
  out
74
73
  end
@@ -84,15 +83,15 @@ module QueueBus
84
83
  protected
85
84
 
86
85
  def self.normalize(val)
87
- val.to_s.gsub(/\W/, "_").downcase
86
+ val.to_s.gsub(/\W/, '_').downcase
88
87
  end
89
88
 
90
89
  def self.app_list_key
91
- "bus_apps"
90
+ 'bus_apps'
92
91
  end
93
92
 
94
93
  def self.app_single_key
95
- "bus_app"
94
+ 'bus_app'
96
95
  end
97
96
 
98
97
  def event_queues
@@ -116,6 +115,5 @@ module QueueBus
116
115
  end
117
116
  out
118
117
  end
119
-
120
118
  end
121
119
  end
@@ -1,102 +1,100 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'socket'
4
+ require 'logger'
5
+
1
6
  module QueueBus
7
+ # This class contains all the configuration for a running queue bus application.
2
8
  class Config
3
- def adapter=val
4
- raise "Adapter already set to #{@adapter_instance.class.name}" if @adapter_instance
5
- if val.is_a?(Class)
6
- @adapter_instance = name_or_klass.new
7
- elsif val.is_a?(::QueueBus::Adapters::Base)
8
- @adapter_instance = val
9
- else
10
- class_name = ::QueueBus::Util.classify(val)
11
- @adapter_instance = ::QueueBus::Util.constantize("::QueueBus::Adapters::#{class_name}").new
12
- end
13
- @adapter_instance
14
- end
9
+ attr_accessor :default_queue, :hostname, :incoming_queue, :logger
15
10
 
16
- def adapter
17
- return @adapter_instance if @adapter_instance
18
- raise "no adapter has been set"
19
- end
11
+ attr_reader :worker_middleware_stack
12
+ attr_writer :local_mode
20
13
 
21
- def redis(&block)
22
- # TODO: could allow setting for non-redis adapters
23
- adapter.redis(&block)
14
+ def initialize
15
+ @worker_middleware_stack = QueueBus::Middleware::Stack.new
16
+ @incoming_queue = 'bus_incoming'
17
+ @hostname = Socket.gethostname
24
18
  end
25
19
 
26
- def default_app_key=val
27
- @default_app_key = Application.normalize(val)
28
- end
20
+ # A wrapper that is always "truthy" but can contain an inner value. This is useful for
21
+ # checking that a thread local variable is set to a value, even if that value happens to
22
+ # be nil. This is important because setting a thread local value to nil will cause it to
23
+ # be deleted.
24
+ Wrap = Struct.new(:value)
29
25
 
30
- def default_app_key
31
- @default_app_key
32
- end
26
+ LOCAL_MODE_VAR = :queue_bus_local_mode
33
27
 
34
- def default_queue=val
35
- @default_queue = val
28
+ # Returns the current local mode of QueueBus
29
+ def local_mode
30
+ if Thread.current.thread_variable_get(LOCAL_MODE_VAR).is_a?(Wrap)
31
+ Thread.current.thread_variable_get(LOCAL_MODE_VAR).value
32
+ else
33
+ @local_mode
34
+ end
36
35
  end
37
36
 
38
- def default_queue
39
- @default_queue
37
+ # Overrides the current local mode for the duration of a block. This is a threadsafe
38
+ # implementation. After, the global setting will be resumed.
39
+ #
40
+ # @param mode [Symbol] the mode to switch to
41
+ def with_local_mode(mode)
42
+ previous = Thread.current.thread_variable_get(LOCAL_MODE_VAR)
43
+ Thread.current.thread_variable_set(LOCAL_MODE_VAR, Wrap.new(mode))
44
+ yield if block_given?
45
+ ensure
46
+ Thread.current.thread_variable_set(LOCAL_MODE_VAR, previous)
40
47
  end
41
48
 
42
- def local_mode=value
43
- @local_mode = value
44
- end
49
+ def adapter=(val)
50
+ raise "Adapter already set to #{@adapter_instance.class.name}" if has_adapter?
45
51
 
46
- def local_mode
47
- @local_mode
52
+ @adapter_instance =
53
+ if val.is_a?(Class)
54
+ val.new
55
+ elsif val.is_a?(::QueueBus::Adapters::Base)
56
+ val
57
+ else
58
+ class_name = ::QueueBus::Util.classify(val)
59
+ ::QueueBus::Util.constantize("::QueueBus::Adapters::#{class_name}").new
60
+ end
48
61
  end
49
62
 
50
- def incoming_queue=val
51
- @incoming_queue = val
52
- end
63
+ def adapter
64
+ return @adapter_instance if has_adapter?
53
65
 
54
- def incoming_queue
55
- @incoming_queue ||= "bus_incoming"
66
+ raise 'no adapter has been set'
56
67
  end
57
68
 
58
- def worker_middleware_stack
59
- @worker_middleware_stack ||= QueueBus::Middleware::Stack.new
69
+ # Checks whether an adapter is set and returns true if it is.
70
+ def has_adapter? # rubocop:disable Naming/PredicateName
71
+ !@adapter_instance.nil?
60
72
  end
61
73
 
62
- def hostname
63
- @hostname ||= `hostname 2>&1`.strip.sub(/.local/,'')
74
+ def redis(&block)
75
+ # TODO: could allow setting for non-redis adapters
76
+ adapter.redis(&block)
64
77
  end
65
78
 
66
- def hostname=val
67
- @hostname = val
79
+ attr_reader :default_app_key
80
+ def default_app_key=(val)
81
+ @default_app_key = Application.normalize(val)
68
82
  end
69
83
 
70
- def before_publish=(proc)
71
- @before_publish_callback = proc
84
+ def before_publish=(callback)
85
+ @before_publish_callback = callback
72
86
  end
73
87
 
74
88
  def before_publish_callback(attributes)
75
- if @before_publish_callback
76
- @before_publish_callback.call(attributes)
77
- end
78
- end
79
-
80
- def logger
81
- @logger
82
- end
83
-
84
- def logger=val
85
- @logger = val
89
+ @before_publish_callback&.call(attributes)
86
90
  end
87
91
 
88
92
  def log_application(message)
89
- if logger
90
- time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
91
- logger.info("** [#{time}] #$$: QueueBus #{message}")
92
- end
93
+ logger&.info(message)
93
94
  end
94
95
 
95
96
  def log_worker(message)
96
- if ENV['LOGGING'] || ENV['VERBOSE'] || ENV['VVERBOSE']
97
- time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
98
- puts "** [#{time}] #$$: #{message}"
99
- end
97
+ logger&.debug(message)
100
98
  end
101
99
  end
102
100
  end