sidekiq-throttled 0.18.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.adoc +314 -0
  3. data/lib/sidekiq/throttled/config.rb +44 -0
  4. data/lib/sidekiq/throttled/cooldown.rb +55 -0
  5. data/lib/sidekiq/throttled/expirable_set.rb +70 -0
  6. data/lib/sidekiq/throttled/job.rb +4 -4
  7. data/lib/sidekiq/throttled/middlewares/server.rb +28 -0
  8. data/lib/sidekiq/throttled/patches/basic_fetch.rb +53 -0
  9. data/lib/sidekiq/throttled/registry.rb +4 -7
  10. data/lib/sidekiq/throttled/strategy/concurrency.rb +2 -4
  11. data/lib/sidekiq/throttled/strategy/threshold.rb +2 -4
  12. data/lib/sidekiq/throttled/strategy.rb +10 -10
  13. data/lib/sidekiq/throttled/strategy_collection.rb +2 -3
  14. data/lib/sidekiq/throttled/version.rb +1 -1
  15. data/lib/sidekiq/throttled/web.rb +2 -45
  16. data/lib/sidekiq/throttled/worker.rb +1 -1
  17. data/lib/sidekiq/throttled.rb +45 -57
  18. metadata +22 -67
  19. data/.coveralls.yml +0 -1
  20. data/.github/dependabot.yml +0 -12
  21. data/.github/workflows/ci.yml +0 -52
  22. data/.gitignore +0 -12
  23. data/.rspec +0 -5
  24. data/.rubocop.yml +0 -20
  25. data/.rubocop_todo.yml +0 -68
  26. data/.travis.yml +0 -37
  27. data/.yardopts +0 -1
  28. data/Appraisals +0 -9
  29. data/CHANGES.md +0 -318
  30. data/Gemfile +0 -34
  31. data/Guardfile +0 -25
  32. data/README.md +0 -297
  33. data/Rakefile +0 -27
  34. data/gemfiles/sidekiq_6.4.gemfile +0 -33
  35. data/gemfiles/sidekiq_6.5.gemfile +0 -33
  36. data/lib/sidekiq/throttled/communicator/callbacks.rb +0 -72
  37. data/lib/sidekiq/throttled/communicator/exception_handler.rb +0 -25
  38. data/lib/sidekiq/throttled/communicator/listener.rb +0 -109
  39. data/lib/sidekiq/throttled/communicator.rb +0 -116
  40. data/lib/sidekiq/throttled/configuration.rb +0 -50
  41. data/lib/sidekiq/throttled/expirable_list.rb +0 -70
  42. data/lib/sidekiq/throttled/fetch/unit_of_work.rb +0 -83
  43. data/lib/sidekiq/throttled/fetch.rb +0 -94
  44. data/lib/sidekiq/throttled/middleware.rb +0 -22
  45. data/lib/sidekiq/throttled/patches/queue.rb +0 -18
  46. data/lib/sidekiq/throttled/queue_name.rb +0 -46
  47. data/lib/sidekiq/throttled/queues_pauser.rb +0 -152
  48. data/lib/sidekiq/throttled/testing.rb +0 -12
  49. data/lib/sidekiq/throttled/utils.rb +0 -19
  50. data/lib/sidekiq/throttled/web/queues.html.erb +0 -49
  51. data/lib/sidekiq/throttled/web/summary_fix.js +0 -10
  52. data/lib/sidekiq/throttled/web/summary_fix.rb +0 -35
  53. data/rubocop/layout.yml +0 -24
  54. data/rubocop/lint.yml +0 -41
  55. data/rubocop/metrics.yml +0 -4
  56. data/rubocop/performance.yml +0 -25
  57. data/rubocop/rspec.yml +0 -3
  58. data/rubocop/style.yml +0 -84
  59. data/sidekiq-throttled.gemspec +0 -36
  60. /data/{LICENSE.md → LICENSE.txt} +0 -0
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # internal
4
- require "sidekiq/throttled/registry"
5
-
6
- module Sidekiq
7
- module Throttled
8
- # Server middleware that notifies strategy that job was finished.
9
- #
10
- # @private
11
- class Middleware
12
- # Called within Sidekiq job processing
13
- def call(_worker, msg, _queue)
14
- yield
15
- ensure
16
- Registry.get msg["class"] do |strategy|
17
- strategy.finalize!(msg["jid"], *msg["args"])
18
- end
19
- end
20
- end
21
- end
22
- end
@@ -1,18 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Sidekiq
4
- module Throttled
5
- module Patches
6
- module Queue
7
- def paused?
8
- QueuesPauser.instance.paused? name
9
- end
10
-
11
- def self.apply!
12
- require "sidekiq/api"
13
- ::Sidekiq::Queue.send(:prepend, self)
14
- end
15
- end
16
- end
17
- end
18
- end
@@ -1,46 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Sidekiq
4
- module Throttled
5
- # Queue name utility belt.
6
- #
7
- # @private
8
- module QueueName
9
- # RegExp used to stip out any redisr-namespace prefixes with `queue:`.
10
- QUEUE_NAME_PREFIX_RE = %r{.*queue:}.freeze
11
- private_constant :QUEUE_NAME_PREFIX_RE
12
-
13
- class << self
14
- # Strips redis-namespace and `queue:` prefix from given queue name.
15
- #
16
- # @example
17
- #
18
- # QueueName.normalize "queue:default"
19
- # # => "default"
20
- #
21
- # QueueName.normalize "queue:queue:default"
22
- # # => "default"
23
- #
24
- # QueueName.normalize "foo:bar:queue:default"
25
- # # => "default"
26
- #
27
- # @param [#to_s]
28
- # @return [String]
29
- def normalize(queue)
30
- -queue.to_s.sub(QUEUE_NAME_PREFIX_RE, "")
31
- end
32
-
33
- # Prepends `queue:` prefix to given `queue` name.
34
- #
35
- # @note It does not normalizes queue before expanding it, thus
36
- # double-call of this method will potentially do some harm.
37
- #
38
- # @param [#to_s] queue Queue name
39
- # @return [String]
40
- def expand(queue)
41
- -"queue:#{queue}"
42
- end
43
- end
44
- end
45
- end
46
- end
@@ -1,152 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "set"
4
- require "singleton"
5
- require "concurrent/timer_task"
6
-
7
- require "sidekiq/throttled/patches/queue"
8
- require "sidekiq/throttled/communicator"
9
- require "sidekiq/throttled/queue_name"
10
-
11
- module Sidekiq
12
- module Throttled
13
- # Singleton class used to pause queues from being processed.
14
- # For the sake of efficiency it uses {Communicator} behind the scene
15
- # to notify all processes about paused/resumed queues.
16
- #
17
- # @private
18
- class QueuesPauser
19
- include Singleton
20
-
21
- # Redis key of Set with paused queues.
22
- #
23
- # @return [String]
24
- PAUSED_QUEUES = "throttled:X:paused_queues"
25
- private_constant :PAUSED_QUEUES
26
-
27
- # {Communicator} message used to notify that queue needs to be paused.
28
- #
29
- # @return [String]
30
- PAUSE_MESSAGE = "pause"
31
- private_constant :PAUSE_MESSAGE
32
-
33
- # {Communicator} message used to notify that queue needs to be resumed.
34
- #
35
- # @return [String]
36
- RESUME_MESSAGE = "resume"
37
- private_constant :RESUME_MESSAGE
38
-
39
- # Initializes singleton instance.
40
- def initialize
41
- @paused_queues = Set.new
42
- @communicator = Communicator.instance
43
- @mutex = Mutex.new
44
- end
45
-
46
- # Configures Sidekiq server to keep actual list of paused queues.
47
- #
48
- # @private
49
- # @return [void]
50
- def setup!
51
- Patches::Queue.apply!
52
-
53
- Sidekiq.configure_server do |config|
54
- config.on(:startup) { start_watcher }
55
- config.on(:quiet) { stop_watcher }
56
-
57
- @communicator.receive(PAUSE_MESSAGE, &method(:add))
58
- @communicator.receive(RESUME_MESSAGE, &method(:delete))
59
- @communicator.ready { sync! }
60
- end
61
- end
62
-
63
- # Returns queues list with paused queues being stripped out.
64
- #
65
- # @private
66
- # @return [Array<String>]
67
- def filter(queues)
68
- @mutex.synchronize { queues - @paused_queues.to_a }
69
- rescue => e
70
- Sidekiq.logger.error { "[#{self.class}] Failed filter queues: #{e}" }
71
- queues
72
- end
73
-
74
- # Returns list of paused queues.
75
- #
76
- # @return [Array<String>]
77
- def paused_queues
78
- Sidekiq.redis { |conn| conn.smembers(PAUSED_QUEUES).to_a }
79
- end
80
-
81
- # Pauses given `queue`.
82
- #
83
- # @param [#to_s] queue
84
- # @return [void]
85
- def pause!(queue)
86
- queue = QueueName.normalize queue.to_s
87
-
88
- Sidekiq.redis do |conn|
89
- conn.sadd(PAUSED_QUEUES, queue)
90
- @communicator.transmit(conn, PAUSE_MESSAGE, queue)
91
- end
92
- end
93
-
94
- # Checks if given `queue` is paused.
95
- #
96
- # @param queue [#to_s]
97
- # @return [Boolean]
98
- def paused?(queue)
99
- queue = QueueName.normalize queue.to_s
100
- Sidekiq.redis { |conn| conn.sismember(PAUSED_QUEUES, queue) }
101
- end
102
-
103
- # Resumes given `queue`.
104
- #
105
- # @param [#to_s] queue
106
- # @return [void]
107
- def resume!(queue)
108
- queue = QueueName.normalize queue.to_s
109
-
110
- Sidekiq.redis do |conn|
111
- conn.srem(PAUSED_QUEUES, queue)
112
- @communicator.transmit(conn, RESUME_MESSAGE, queue)
113
- end
114
- end
115
-
116
- private
117
-
118
- def add(queue)
119
- @mutex.synchronize do
120
- @paused_queues << QueueName.expand(queue)
121
- end
122
- end
123
-
124
- def delete(queue)
125
- @mutex.synchronize do
126
- @paused_queues.delete QueueName.expand(queue)
127
- end
128
- end
129
-
130
- def sync!
131
- @mutex.synchronize do
132
- @paused_queues.replace(paused_queues.map { |q| QueueName.expand q })
133
- end
134
- end
135
-
136
- def start_watcher
137
- @mutex.synchronize do
138
- @watcher ||= Concurrent::TimerTask.execute({
139
- :run_now => true,
140
- :execution_interval => 60
141
- }) { sync! }
142
- end
143
- end
144
-
145
- def stop_watcher
146
- @mutex.synchronize do
147
- defined?(@watcher) && @watcher&.shutdown
148
- end
149
- end
150
- end
151
- end
152
- end
@@ -1,12 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "sidekiq/throttled/registry"
4
-
5
- RSpec.configure do |config|
6
- config.before :example do
7
- Sidekiq::Throttled::Registry.instance_eval do
8
- @strategies.clear
9
- @aliases.clear
10
- end
11
- end
12
- end
@@ -1,19 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Sidekiq
4
- module Throttled
5
- module Utils
6
- module_function
7
-
8
- # Resolve constant from it's name
9
- # @param name [#to_s] Constant name
10
- # @return [Object, nil] Resolved constant or nil if failed.
11
- def constantize(name)
12
- name.to_s.sub(%r{^::}, "").split("::").inject(Object, &:const_get)
13
- rescue NameError
14
- Sidekiq.logger.warn { "Failed to constantize: #{name}" }
15
- nil
16
- end
17
- end
18
- end
19
- end
@@ -1,49 +0,0 @@
1
- <h3><%= t('Queues') %></h3>
2
-
3
- <div class="table_container">
4
- <table class="queues table table-hover table-bordered table-striped table-white">
5
- <thead>
6
- <th><%= t('Queue') %></th>
7
- <th><%= t('Size') %></th>
8
- <th><%= t('Actions') %></th>
9
- </thead>
10
- <% @queues.each do |queue| %>
11
- <tr>
12
- <td>
13
- <a href="<%= root_path %>queues/<%= CGI.escape(queue.name) %>"><%= queue.name %></a>
14
- </td>
15
- <td><%= number_with_delimiter(queue.size) %> </td>
16
- <td width="20%">
17
- <form action="<%=root_path %>enhanced-queues/<%= CGI.escape(queue.name) %>" method="post" style="display:inline">
18
- <%= csrf_tag %>
19
- <% if queue.paused? %>
20
- <button class="btn btn-primary btn-pauser-resume btn-xs" type="submit" name="action" value="resume">Resume</button>
21
- <% else %>
22
- <button class="btn btn-danger btn-pauser-pause btn-xs" type="submit" name="action" value="pause">Pause</button>
23
- <% end %>
24
-
25
- <button class="btn btn-danger btn-xs" type="submit" name="action" value="delete" data-confirm="<%= t('AreYouSureDeleteQueue', :queue => h(queue.name)) %>"><%= t('Delete') %></button>
26
- </form>
27
- </td>
28
- </tr>
29
- <% end %>
30
- </table>
31
- </div>
32
-
33
- <style>
34
- .btn-pauser-pause {
35
- background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #b13e00), color-stop(100%, #983500));
36
- background-image: -webkit-linear-gradient(#b13e00, #983500);
37
- background-image: -moz-linear-gradient(#b13e00, #983500);
38
- background-image: -o-linear-gradient(#b13e00, #983500);
39
- background-image: linear-gradient(#b13e00, #983500);
40
- }
41
-
42
- .btn-pauser-resume {
43
- background-image: -webkit-gradient(linear, 50% 0%, 50% 100%, color-stop(0%, #00b13e), color-stop(100%, #009835));
44
- background-image: -webkit-linear-gradient(#00b13e, #009835);
45
- background-image: -moz-linear-gradient(#00b13e, #009835);
46
- background-image: -o-linear-gradient(#00b13e, #009835);
47
- background-image: linear-gradient(#00b13e, #009835);
48
- }
49
- </style>
@@ -1,10 +0,0 @@
1
- document.addEventListener("DOMContentLoaded", function () {
2
- var elem = document.querySelector(".summary li.enqueued > a"), href;
3
-
4
- if (!elem) {
5
- console.warn("[Sidekiq::Threshold] cannot find summary bar link to fix");
6
- } else {
7
- href = elem.getAttribute("href").toString();
8
- elem.setAttribute("href", href.replace(/\/queues$/, "/enhanced-queues"));
9
- }
10
- });
@@ -1,35 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Sidekiq
4
- module Throttled
5
- module Web
6
- module SummaryFix
7
- JAVASCRIPT = [File.read(File.expand_path("summary_fix.js", __dir__)).freeze].freeze
8
- HEADERS = { "Content-Type" => "application/javascript" }.freeze
9
-
10
- class << self
11
- attr_accessor :enabled
12
-
13
- def apply!(app)
14
- Sidekiq::WebAction.prepend SummaryFix
15
-
16
- app.get("/throttled/summary_fix") do
17
- [200, HEADERS.dup, JAVASCRIPT.dup]
18
- end
19
- end
20
- end
21
-
22
- def display_custom_head
23
- "#{super}#{summary_fix_script if SummaryFix.enabled}"
24
- end
25
-
26
- private
27
-
28
- def summary_fix_script
29
- src = "#{root_path}throttled/summary_fix"
30
- %(<script type="text/javascript" src="#{src}"></script>)
31
- end
32
- end
33
- end
34
- end
35
- end
data/rubocop/layout.yml DELETED
@@ -1,24 +0,0 @@
1
- Layout/ArgumentAlignment:
2
- EnforcedStyle: with_fixed_indentation
3
-
4
- Layout/EmptyLinesAroundAttributeAccessor:
5
- Enabled: true
6
-
7
- Layout/FirstArrayElementIndentation:
8
- EnforcedStyle: consistent
9
-
10
- Layout/FirstHashElementIndentation:
11
- Enabled: true
12
- EnforcedStyle: consistent
13
-
14
- Layout/HashAlignment:
15
- Enabled: true
16
- EnforcedHashRocketStyle: table
17
- EnforcedColonStyle: table
18
-
19
- Layout/SpaceAroundMethodCallOperator:
20
- Enabled: true
21
-
22
- Layout/MultilineMethodCallIndentation:
23
- Enabled: true
24
- EnforcedStyle: indented
data/rubocop/lint.yml DELETED
@@ -1,41 +0,0 @@
1
- Lint/BinaryOperatorWithIdenticalOperands:
2
- Enabled: true
3
-
4
- Lint/DeprecatedOpenSSLConstant:
5
- Enabled: true
6
-
7
- Lint/DuplicateElsifCondition:
8
- Enabled: true
9
-
10
- Lint/DuplicateRescueException:
11
- Enabled: true
12
-
13
- Lint/EmptyConditionalBody:
14
- Enabled: true
15
-
16
- Lint/FloatComparison:
17
- Enabled: true
18
-
19
- Lint/MissingSuper:
20
- Enabled: true
21
-
22
- Lint/MixedRegexpCaptureTypes:
23
- Enabled: true
24
-
25
- Lint/OutOfRangeRegexpRef:
26
- Enabled: true
27
-
28
- Lint/RaiseException:
29
- Enabled: true
30
-
31
- Lint/SelfAssignment:
32
- Enabled: true
33
-
34
- Lint/StructNewOverride:
35
- Enabled: true
36
-
37
- Lint/TopLevelReturnWithArgument:
38
- Enabled: true
39
-
40
- Lint/UnreachableLoop:
41
- Enabled: true
data/rubocop/metrics.yml DELETED
@@ -1,4 +0,0 @@
1
- Metrics/BlockLength:
2
- Exclude:
3
- - "spec/**/*_spec.rb"
4
- - "**/*.gemspec"
@@ -1,25 +0,0 @@
1
- Performance/AncestorsInclude:
2
- Enabled: true
3
-
4
- Performance/BigDecimalWithNumericArgument:
5
- Enabled: true
6
-
7
- Performance/RedundantSortBlock:
8
- Enabled: true
9
-
10
- Performance/RedundantStringChars:
11
- Enabled: true
12
-
13
- Performance/ReverseFirst:
14
- Enabled: true
15
-
16
- Performance/SortReverse:
17
- Enabled: true
18
-
19
- Performance/Squeeze:
20
- Enabled: true
21
-
22
- Performance/StringInclude:
23
- Enabled: true
24
-
25
-
data/rubocop/rspec.yml DELETED
@@ -1,3 +0,0 @@
1
- RSpec/ExampleLength:
2
- Enabled: true
3
- Max: 15
data/rubocop/style.yml DELETED
@@ -1,84 +0,0 @@
1
- Style/AccessorGrouping:
2
- Enabled: true
3
-
4
- Style/ArrayCoercion:
5
- Enabled: true
6
-
7
- Style/BisectedAttrAccessor:
8
- Enabled: true
9
-
10
- Style/CaseLikeIf:
11
- Enabled: true
12
-
13
- Style/Documentation:
14
- Enabled: false
15
-
16
- Style/ExplicitBlockArgument:
17
- Enabled: true
18
-
19
- Style/ExponentialNotation:
20
- Enabled: true
21
-
22
- Style/GlobalStdStream:
23
- Enabled: true
24
-
25
- Style/HashAsLastArrayItem:
26
- Enabled: true
27
-
28
- Style/HashEachMethods:
29
- Enabled: true
30
-
31
- Style/HashLikeCase:
32
- Enabled: true
33
-
34
- Style/HashSyntax:
35
- Enabled: true
36
- EnforcedStyle: hash_rockets
37
-
38
- Style/HashTransformKeys:
39
- Enabled: true
40
-
41
- Style/HashTransformValues:
42
- Enabled: true
43
-
44
- Style/OptionalBooleanParameter:
45
- Enabled: true
46
-
47
- Style/RedundantAssignment:
48
- Enabled: true
49
-
50
- Style/RedundantFetchBlock:
51
- Enabled: true
52
-
53
- Style/RedundantFileExtensionInRequire:
54
- Enabled: true
55
-
56
- Style/RedundantRegexpCharacterClass:
57
- Enabled: true
58
-
59
- Style/RedundantRegexpEscape:
60
- Enabled: true
61
-
62
- Style/RegexpLiteral:
63
- Enabled: true
64
- EnforcedStyle: percent_r
65
-
66
- Style/RescueStandardError:
67
- Enabled: true
68
- EnforcedStyle: implicit
69
-
70
- Style/SingleArgumentDig:
71
- Enabled: true
72
-
73
- Style/SlicingWithRange:
74
- Enabled: true
75
-
76
- Style/StringConcatenation:
77
- Enabled: true
78
-
79
- Style/StringLiterals:
80
- Enabled: true
81
- EnforcedStyle: double_quotes
82
-
83
- Style/YodaCondition:
84
- Enabled: false
@@ -1,36 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- lib = File.expand_path("lib", __dir__)
4
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
-
6
- require "sidekiq/throttled/version"
7
-
8
- Gem::Specification.new do |spec|
9
- spec.name = "sidekiq-throttled"
10
- spec.version = Sidekiq::Throttled::VERSION
11
- spec.authors = ["Alexey Zapparov"]
12
- spec.email = ["alexey@zapparov.com"]
13
-
14
- spec.summary = "Concurrency and threshold throttling for Sidekiq."
15
- spec.description = "Concurrency and threshold throttling for Sidekiq."
16
- spec.homepage = "https://github.com/ixti/sidekiq-throttled"
17
- spec.license = "MIT"
18
-
19
- spec.files = `git ls-files -z`.split("\x0").reject do |f|
20
- f.match %r{^(test|spec|features)/}
21
- end
22
-
23
- spec.metadata["rubygems_mfa_required"] = "true"
24
-
25
- spec.bindir = "exe"
26
- spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
27
- spec.require_paths = ["lib"]
28
-
29
- spec.required_ruby_version = ">= 2.7"
30
-
31
- spec.add_runtime_dependency "concurrent-ruby"
32
- spec.add_runtime_dependency "redis-prescription", "~> 2.2"
33
- spec.add_runtime_dependency "sidekiq", "~> 6.4"
34
-
35
- spec.add_development_dependency "bundler", ">= 2.0"
36
- end
File without changes