sidekiq-throttled 0.6.1 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 62a9ad77603d8384fb1b5f3646e68242a7e47e6a
4
- data.tar.gz: ce6e7f785851824533d6657779f2d778805802e1
3
+ metadata.gz: 781ea0cbad892bcfe6e726f96247ea457e07a824
4
+ data.tar.gz: 69cecb30661c7527e56d30c79bb454e6aa79f51f
5
5
  SHA512:
6
- metadata.gz: cd27f6c8afda43c910319ffd049f9961a86c76170e820bca7d0e15e39f376fe13e83fd34bae141b4ec8b46d7b0106a3b048faed60a3f0c5618d164cfdfe6b902
7
- data.tar.gz: 36b7f0c2232ef68fe7eeaa90044234a21b96fe974628b5cc6f96e7e1f2f8b05655b8ab7b531eec0a8d02533927da4dc49cec689549bd1bf53c55b1a2687b25b9
6
+ metadata.gz: 6c6126e4d3b38713f8a57cc614b3fa7d9ec06a4bd600c9c59ba3bc51ca4ae1d7441dedaca184cda92f064e1f18b9f495c0c022a62495b3212fcf5f3cf7095ffc
7
+ data.tar.gz: c639b892aacc37c5f47cb888cfd27656258ec08ac5e510cfdbdf719dd19477d57505067a14ac1798505ae03e4ad3a860c0a6e2ee6fa1d1db61ffe6d99bd8ecd0
data/CHANGES.md CHANGED
@@ -1,6 +1,12 @@
1
+ ## 0.6.2 (2016-09-01)
2
+
3
+ * Add `Fetch.bulk_requeue` used by Sidekiq upon termination.
4
+ ([@ixti])
5
+
6
+
1
7
  ## 0.6.1 (2016-08-30)
2
8
 
3
- * Trivial internbal API change: extracted queues list builder of `Fetch` into
9
+ * Trivial internal API change: extracted queues list builder of `Fetch` into
4
10
  dedicated internal method, allowing to enhance it with extra custom filters.
5
11
  ([@ixti])
6
12
 
data/LICENSE.md CHANGED
@@ -1,6 +1,6 @@
1
1
  The MIT License (MIT)
2
2
 
3
- Copyright (c) 2015 SensorTower Inc.
3
+ Copyright (c) 2015-2016 SensorTower Inc.
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
data/README.md CHANGED
@@ -4,6 +4,7 @@
4
4
  [![Build Status](https://travis-ci.org/sensortower/sidekiq-throttled.svg?branch=master)](https://travis-ci.org/sensortower/sidekiq-throttled)
5
5
  [![Code Climate](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
6
6
  [![Coverage Status](https://coveralls.io/repos/sensortower/sidekiq-throttled/badge.svg?branch=master&service=github)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
7
+ [![API Docs](http://inch-ci.org/github/sensortower/sidekiq-throttled.svg?branch=master)](http://inch-ci.org/github/sensortower/sidekiq-throttled)
7
8
 
8
9
  Concurrency and threshold throttling for [Sidekiq][sidekiq].
9
10
 
@@ -145,7 +146,7 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
145
146
 
146
147
  ## Copyright
147
148
 
148
- Copyright (c) 2015 SensorTower Inc.
149
+ Copyright (c) 2015-2016 SensorTower Inc.
149
150
  See LICENSE.md for further details.
150
151
 
151
152
 
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq"
4
- require "sidekiq/throttled/unit_of_work"
4
+ require "sidekiq/throttled/fetch/unit_of_work"
5
5
  require "sidekiq/throttled/queues_pauser"
6
6
  require "sidekiq/throttled/queue_name"
7
7
 
@@ -22,6 +22,8 @@ module Sidekiq
22
22
  @queues.uniq! if @strict
23
23
  end
24
24
 
25
+ # Retrieves job from redis.
26
+ #
25
27
  # @return [Sidekiq::Throttled::UnitOfWork, nil]
26
28
  def retrieve_work
27
29
  work = brpop
@@ -30,21 +32,36 @@ module Sidekiq
30
32
  work = UnitOfWork.new(*work)
31
33
  return work unless Throttled.throttled? work.job
32
34
 
33
- Sidekiq.redis do |conn|
34
- conn.lpush(QueueName.expand(work.queue_name), work.job)
35
- end
35
+ work.throttled_requeue
36
36
 
37
37
  nil
38
38
  end
39
39
 
40
+ class << self
41
+ # Requeues all given units as a single operation.
42
+ #
43
+ # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
44
+ # @param [Array<Fetch::UnitOfWork>] units
45
+ # @return [void]
46
+ def bulk_requeue(units, _options)
47
+ return if units.empty?
48
+
49
+ Sidekiq.logger.debug { "Re-queueing terminated jobs" }
50
+ Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
51
+ Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
52
+ rescue => e
53
+ Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
54
+ end
55
+ end
56
+
40
57
  private
41
58
 
42
- # Tries to pop pair of `queue` and job `message` out of sidekiq queue.
59
+ # Tries to pop pair of `queue` and job `message` out of sidekiq queues.
43
60
  #
44
61
  # @see http://redis.io/commands/brpop
45
62
  # @return [Array(String, String), nil]
46
63
  def brpop
47
- queues = build_queues_list
64
+ queues = filter_queues(@strict ? @queues : @queues.shuffle.uniq)
48
65
 
49
66
  if queues.empty?
50
67
  sleep TIMEOUT
@@ -57,9 +74,10 @@ module Sidekiq
57
74
  # Returns list of queues to try to fetch jobs from.
58
75
  #
59
76
  # @note It may return an empty array.
77
+ # @param [Array<String>] queues
60
78
  # @return [Array<String>]
61
- def build_queues_list
62
- QueuesPauser.instance.filter(@strict ? @queues : @queues.shuffle.uniq)
79
+ def filter_queues(queues)
80
+ QueuesPauser.instance.filter(queues)
63
81
  end
64
82
  end
65
83
  end
@@ -0,0 +1,70 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq"
4
+
5
+ require "sidekiq/throttled/queue_name"
6
+
7
+ module Sidekiq
8
+ module Throttled
9
+ class Fetch
10
+ # BRPOP response envelope.
11
+ #
12
+ # @see Throttled::Fetch
13
+ # @private
14
+ class UnitOfWork
15
+ # @return [String] Redis key where job was pulled from
16
+ attr_reader :queue
17
+
18
+ # @return [String] Job's JSON payload
19
+ attr_reader :job
20
+
21
+ # @param [String] queue Redis key where job was pulled from
22
+ # @param [String] job Job's JSON payload
23
+ def initialize(queue, job)
24
+ @queue = queue
25
+ @job = job
26
+ end
27
+
28
+ # Callback that is called by `Sidekiq::Processor` when job was
29
+ # succeccfully processed. Most this is used by `ReliableFetch`
30
+ # of Sidekiq Pro/Enterprise to remove job from running queue.
31
+ #
32
+ # @return [void]
33
+ def acknowledge
34
+ # do nothing
35
+ end
36
+
37
+ # Normalized `queue` name.
38
+ #
39
+ # @see QueueName.normalize
40
+ # @return [String]
41
+ def queue_name
42
+ @queue_name ||= QueueName.normalize queue
43
+ end
44
+
45
+ # Pushes job back to the tail of the queue, so that it will be popped
46
+ # first next time fetcher will pull job.
47
+ #
48
+ # @note This is triggered when job was not finished and Sidekiq server
49
+ # process was terminated. It is a reverse of whatever fetcher was
50
+ # doing to pull the job out of queue.
51
+ #
52
+ # @return [void]
53
+ def requeue
54
+ Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
55
+ end
56
+
57
+ # Pushes job back to the head of the queue, so that job won't be tried
58
+ # immediately after it was requeued (in most cases).
59
+ #
60
+ # @note This is triggered when job is throttled. So it is same operation
61
+ # Sidekiq performs upon `Sidekiq::Worker.perform_async` call.
62
+ #
63
+ # @return [void]
64
+ def throttled_requeue
65
+ Sidekiq.redis { |conn| conn.lpush(QueueName.expand(queue_name), job) }
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -3,6 +3,6 @@
3
3
  module Sidekiq
4
4
  module Throttled
5
5
  # Gem version
6
- VERSION = "0.6.1".freeze
6
+ VERSION = "0.6.2".freeze
7
7
  end
8
8
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-throttled
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 0.6.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alexey V Zapparov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2016-08-30 00:00:00.000000000 Z
11
+ date: 2016-09-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sidekiq
@@ -66,6 +66,7 @@ files:
66
66
  - lib/sidekiq/throttled/communicator/listener.rb
67
67
  - lib/sidekiq/throttled/errors.rb
68
68
  - lib/sidekiq/throttled/fetch.rb
69
+ - lib/sidekiq/throttled/fetch/unit_of_work.rb
69
70
  - lib/sidekiq/throttled/middleware.rb
70
71
  - lib/sidekiq/throttled/queue_name.rb
71
72
  - lib/sidekiq/throttled/queues_pauser.rb
@@ -77,7 +78,6 @@ files:
77
78
  - lib/sidekiq/throttled/strategy/threshold.lua
78
79
  - lib/sidekiq/throttled/strategy/threshold.rb
79
80
  - lib/sidekiq/throttled/testing.rb
80
- - lib/sidekiq/throttled/unit_of_work.rb
81
81
  - lib/sidekiq/throttled/version.rb
82
82
  - lib/sidekiq/throttled/web.rb
83
83
  - lib/sidekiq/throttled/web/index.html.erb
@@ -1,56 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "sidekiq"
4
-
5
- require "sidekiq/throttled/queue_name"
6
-
7
- module Sidekiq
8
- module Throttled
9
- # BRPOP response envelope.
10
- #
11
- # @see Throttled::Fetch
12
- # @private
13
- class UnitOfWork
14
- # @return [String] Redis key where job was pulled from
15
- attr_reader :queue
16
-
17
- # @return [String] Job's JSON payload
18
- attr_reader :job
19
-
20
- # @param [String] queue Redis key where job was pulled from
21
- # @param [String] job Job's JSON payload
22
- def initialize(queue, job)
23
- @queue = queue
24
- @job = job
25
- end
26
-
27
- # Callback that is called by `Sidekiq::Processor` when job was
28
- # succeccfully processed. Most this is used by `ReliableFetch`
29
- # of Sidekiq Pro/Enterprise to remove job from running queue.
30
- #
31
- # @return [void]
32
- def acknowledge
33
- # do nothing
34
- end
35
-
36
- # Normalized `queue` name.
37
- #
38
- # @see QueueName.normalize
39
- # @return [String]
40
- def queue_name
41
- @queue_name ||= QueueName.normalize queue
42
- end
43
-
44
- # Pushes job back to the queue.
45
- #
46
- # @note This is triggered when job was not finished and Sidekiq server
47
- # process was terminated (shutdowned). Thus it should be reverse of
48
- # whatever fetcher was doing to pull the job out of queue.
49
- #
50
- # @return [void]
51
- def requeue
52
- Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
53
- end
54
- end
55
- end
56
- end