sidekiq-throttled 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +7 -1
- data/LICENSE.md +1 -1
- data/README.md +2 -1
- data/lib/sidekiq/throttled/fetch.rb +26 -8
- data/lib/sidekiq/throttled/fetch/unit_of_work.rb +70 -0
- data/lib/sidekiq/throttled/version.rb +1 -1
- metadata +3 -3
- data/lib/sidekiq/throttled/unit_of_work.rb +0 -56
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 781ea0cbad892bcfe6e726f96247ea457e07a824
|
4
|
+
data.tar.gz: 69cecb30661c7527e56d30c79bb454e6aa79f51f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c6126e4d3b38713f8a57cc614b3fa7d9ec06a4bd600c9c59ba3bc51ca4ae1d7441dedaca184cda92f064e1f18b9f495c0c022a62495b3212fcf5f3cf7095ffc
|
7
|
+
data.tar.gz: c639b892aacc37c5f47cb888cfd27656258ec08ac5e510cfdbdf719dd19477d57505067a14ac1798505ae03e4ad3a860c0a6e2ee6fa1d1db61ffe6d99bd8ecd0
|
data/CHANGES.md
CHANGED
@@ -1,6 +1,12 @@
|
|
1
|
+
## 0.6.2 (2016-09-01)
|
2
|
+
|
3
|
+
* Add `Fetch.bulk_requeue` used by Sidekiq upon termination.
|
4
|
+
([@ixti])
|
5
|
+
|
6
|
+
|
1
7
|
## 0.6.1 (2016-08-30)
|
2
8
|
|
3
|
-
* Trivial
|
9
|
+
* Trivial internal API change: extracted queues list builder of `Fetch` into
|
4
10
|
dedicated internal method, allowing to enhance it with extra custom filters.
|
5
11
|
([@ixti])
|
6
12
|
|
data/LICENSE.md
CHANGED
data/README.md
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
[](https://travis-ci.org/sensortower/sidekiq-throttled)
|
5
5
|
[](https://codeclimate.com/github/sensortower/sidekiq-throttled)
|
6
6
|
[](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
|
7
|
+
[](http://inch-ci.org/github/sensortower/sidekiq-throttled)
|
7
8
|
|
8
9
|
Concurrency and threshold throttling for [Sidekiq][sidekiq].
|
9
10
|
|
@@ -145,7 +146,7 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
|
|
145
146
|
|
146
147
|
## Copyright
|
147
148
|
|
148
|
-
Copyright (c) 2015 SensorTower Inc.
|
149
|
+
Copyright (c) 2015-2016 SensorTower Inc.
|
149
150
|
See LICENSE.md for further details.
|
150
151
|
|
151
152
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "sidekiq"
|
4
|
-
require "sidekiq/throttled/unit_of_work"
|
4
|
+
require "sidekiq/throttled/fetch/unit_of_work"
|
5
5
|
require "sidekiq/throttled/queues_pauser"
|
6
6
|
require "sidekiq/throttled/queue_name"
|
7
7
|
|
@@ -22,6 +22,8 @@ module Sidekiq
|
|
22
22
|
@queues.uniq! if @strict
|
23
23
|
end
|
24
24
|
|
25
|
+
# Retrieves job from redis.
|
26
|
+
#
|
25
27
|
# @return [Sidekiq::Throttled::UnitOfWork, nil]
|
26
28
|
def retrieve_work
|
27
29
|
work = brpop
|
@@ -30,21 +32,36 @@ module Sidekiq
|
|
30
32
|
work = UnitOfWork.new(*work)
|
31
33
|
return work unless Throttled.throttled? work.job
|
32
34
|
|
33
|
-
|
34
|
-
conn.lpush(QueueName.expand(work.queue_name), work.job)
|
35
|
-
end
|
35
|
+
work.throttled_requeue
|
36
36
|
|
37
37
|
nil
|
38
38
|
end
|
39
39
|
|
40
|
+
class << self
|
41
|
+
# Requeues all given units as a single operation.
|
42
|
+
#
|
43
|
+
# @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
|
44
|
+
# @param [Array<Fetch::UnitOfWork>] units
|
45
|
+
# @return [void]
|
46
|
+
def bulk_requeue(units, _options)
|
47
|
+
return if units.empty?
|
48
|
+
|
49
|
+
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
50
|
+
Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
|
51
|
+
Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
|
52
|
+
rescue => e
|
53
|
+
Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
40
57
|
private
|
41
58
|
|
42
|
-
# Tries to pop pair of `queue` and job `message` out of sidekiq
|
59
|
+
# Tries to pop pair of `queue` and job `message` out of sidekiq queues.
|
43
60
|
#
|
44
61
|
# @see http://redis.io/commands/brpop
|
45
62
|
# @return [Array(String, String), nil]
|
46
63
|
def brpop
|
47
|
-
queues =
|
64
|
+
queues = filter_queues(@strict ? @queues : @queues.shuffle.uniq)
|
48
65
|
|
49
66
|
if queues.empty?
|
50
67
|
sleep TIMEOUT
|
@@ -57,9 +74,10 @@ module Sidekiq
|
|
57
74
|
# Returns list of queues to try to fetch jobs from.
|
58
75
|
#
|
59
76
|
# @note It may return an empty array.
|
77
|
+
# @param [Array<String>] queues
|
60
78
|
# @return [Array<String>]
|
61
|
-
def
|
62
|
-
QueuesPauser.instance.filter(
|
79
|
+
def filter_queues(queues)
|
80
|
+
QueuesPauser.instance.filter(queues)
|
63
81
|
end
|
64
82
|
end
|
65
83
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require "sidekiq/throttled/queue_name"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Throttled
|
9
|
+
class Fetch
|
10
|
+
# BRPOP response envelope.
|
11
|
+
#
|
12
|
+
# @see Throttled::Fetch
|
13
|
+
# @private
|
14
|
+
class UnitOfWork
|
15
|
+
# @return [String] Redis key where job was pulled from
|
16
|
+
attr_reader :queue
|
17
|
+
|
18
|
+
# @return [String] Job's JSON payload
|
19
|
+
attr_reader :job
|
20
|
+
|
21
|
+
# @param [String] queue Redis key where job was pulled from
|
22
|
+
# @param [String] job Job's JSON payload
|
23
|
+
def initialize(queue, job)
|
24
|
+
@queue = queue
|
25
|
+
@job = job
|
26
|
+
end
|
27
|
+
|
28
|
+
# Callback that is called by `Sidekiq::Processor` when job was
|
29
|
+
# succeccfully processed. Most this is used by `ReliableFetch`
|
30
|
+
# of Sidekiq Pro/Enterprise to remove job from running queue.
|
31
|
+
#
|
32
|
+
# @return [void]
|
33
|
+
def acknowledge
|
34
|
+
# do nothing
|
35
|
+
end
|
36
|
+
|
37
|
+
# Normalized `queue` name.
|
38
|
+
#
|
39
|
+
# @see QueueName.normalize
|
40
|
+
# @return [String]
|
41
|
+
def queue_name
|
42
|
+
@queue_name ||= QueueName.normalize queue
|
43
|
+
end
|
44
|
+
|
45
|
+
# Pushes job back to the tail of the queue, so that it will be popped
|
46
|
+
# first next time fetcher will pull job.
|
47
|
+
#
|
48
|
+
# @note This is triggered when job was not finished and Sidekiq server
|
49
|
+
# process was terminated. It is a reverse of whatever fetcher was
|
50
|
+
# doing to pull the job out of queue.
|
51
|
+
#
|
52
|
+
# @return [void]
|
53
|
+
def requeue
|
54
|
+
Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
|
55
|
+
end
|
56
|
+
|
57
|
+
# Pushes job back to the head of the queue, so that job won't be tried
|
58
|
+
# immediately after it was requeued (in most cases).
|
59
|
+
#
|
60
|
+
# @note This is triggered when job is throttled. So it is same operation
|
61
|
+
# Sidekiq performs upon `Sidekiq::Worker.perform_async` call.
|
62
|
+
#
|
63
|
+
# @return [void]
|
64
|
+
def throttled_requeue
|
65
|
+
Sidekiq.redis { |conn| conn.lpush(QueueName.expand(queue_name), job) }
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-throttled
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexey V Zapparov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-09-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sidekiq
|
@@ -66,6 +66,7 @@ files:
|
|
66
66
|
- lib/sidekiq/throttled/communicator/listener.rb
|
67
67
|
- lib/sidekiq/throttled/errors.rb
|
68
68
|
- lib/sidekiq/throttled/fetch.rb
|
69
|
+
- lib/sidekiq/throttled/fetch/unit_of_work.rb
|
69
70
|
- lib/sidekiq/throttled/middleware.rb
|
70
71
|
- lib/sidekiq/throttled/queue_name.rb
|
71
72
|
- lib/sidekiq/throttled/queues_pauser.rb
|
@@ -77,7 +78,6 @@ files:
|
|
77
78
|
- lib/sidekiq/throttled/strategy/threshold.lua
|
78
79
|
- lib/sidekiq/throttled/strategy/threshold.rb
|
79
80
|
- lib/sidekiq/throttled/testing.rb
|
80
|
-
- lib/sidekiq/throttled/unit_of_work.rb
|
81
81
|
- lib/sidekiq/throttled/version.rb
|
82
82
|
- lib/sidekiq/throttled/web.rb
|
83
83
|
- lib/sidekiq/throttled/web/index.html.erb
|
@@ -1,56 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "sidekiq"
|
4
|
-
|
5
|
-
require "sidekiq/throttled/queue_name"
|
6
|
-
|
7
|
-
module Sidekiq
|
8
|
-
module Throttled
|
9
|
-
# BRPOP response envelope.
|
10
|
-
#
|
11
|
-
# @see Throttled::Fetch
|
12
|
-
# @private
|
13
|
-
class UnitOfWork
|
14
|
-
# @return [String] Redis key where job was pulled from
|
15
|
-
attr_reader :queue
|
16
|
-
|
17
|
-
# @return [String] Job's JSON payload
|
18
|
-
attr_reader :job
|
19
|
-
|
20
|
-
# @param [String] queue Redis key where job was pulled from
|
21
|
-
# @param [String] job Job's JSON payload
|
22
|
-
def initialize(queue, job)
|
23
|
-
@queue = queue
|
24
|
-
@job = job
|
25
|
-
end
|
26
|
-
|
27
|
-
# Callback that is called by `Sidekiq::Processor` when job was
|
28
|
-
# succeccfully processed. Most this is used by `ReliableFetch`
|
29
|
-
# of Sidekiq Pro/Enterprise to remove job from running queue.
|
30
|
-
#
|
31
|
-
# @return [void]
|
32
|
-
def acknowledge
|
33
|
-
# do nothing
|
34
|
-
end
|
35
|
-
|
36
|
-
# Normalized `queue` name.
|
37
|
-
#
|
38
|
-
# @see QueueName.normalize
|
39
|
-
# @return [String]
|
40
|
-
def queue_name
|
41
|
-
@queue_name ||= QueueName.normalize queue
|
42
|
-
end
|
43
|
-
|
44
|
-
# Pushes job back to the queue.
|
45
|
-
#
|
46
|
-
# @note This is triggered when job was not finished and Sidekiq server
|
47
|
-
# process was terminated (shutdowned). Thus it should be reverse of
|
48
|
-
# whatever fetcher was doing to pull the job out of queue.
|
49
|
-
#
|
50
|
-
# @return [void]
|
51
|
-
def requeue
|
52
|
-
Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|