sidekiq-throttled 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62a9ad77603d8384fb1b5f3646e68242a7e47e6a
-  data.tar.gz: ce6e7f785851824533d6657779f2d778805802e1
+  metadata.gz: 781ea0cbad892bcfe6e726f96247ea457e07a824
+  data.tar.gz: 69cecb30661c7527e56d30c79bb454e6aa79f51f
 SHA512:
-  metadata.gz: cd27f6c8afda43c910319ffd049f9961a86c76170e820bca7d0e15e39f376fe13e83fd34bae141b4ec8b46d7b0106a3b048faed60a3f0c5618d164cfdfe6b902
-  data.tar.gz: 36b7f0c2232ef68fe7eeaa90044234a21b96fe974628b5cc6f96e7e1f2f8b05655b8ab7b531eec0a8d02533927da4dc49cec689549bd1bf53c55b1a2687b25b9
+  metadata.gz: 6c6126e4d3b38713f8a57cc614b3fa7d9ec06a4bd600c9c59ba3bc51ca4ae1d7441dedaca184cda92f064e1f18b9f495c0c022a62495b3212fcf5f3cf7095ffc
+  data.tar.gz: c639b892aacc37c5f47cb888cfd27656258ec08ac5e510cfdbdf719dd19477d57505067a14ac1798505ae03e4ad3a860c0a6e2ee6fa1d1db61ffe6d99bd8ecd0

data/CHANGES.md

@@ -1,6 +1,12 @@
+## 0.6.2 (2016-09-01)
+
+* Add `Fetch.bulk_requeue` used by Sidekiq upon termination.
+  ([@ixti])
+
+
 ## 0.6.1 (2016-08-30)
 
-* Trivial internbal API change: extracted queues list builder of `Fetch` into
+* Trivial internal API change: extracted queues list builder of `Fetch` into
   dedicated internal method, allowing to enhance it with extra custom filters.
   ([@ixti])
 

data/LICENSE.md

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2015 SensorTower Inc.
+Copyright (c) 2015-2016 SensorTower Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -4,6 +4,7 @@
 [![Build Status](https://travis-ci.org/sensortower/sidekiq-throttled.svg?branch=master)](https://travis-ci.org/sensortower/sidekiq-throttled)
 [![Code Climate](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
 [![Coverage Status](https://coveralls.io/repos/sensortower/sidekiq-throttled/badge.svg?branch=master&service=github)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
+[![API Docs](http://inch-ci.org/github/sensortower/sidekiq-throttled.svg?branch=master)](http://inch-ci.org/github/sensortower/sidekiq-throttled)
 
 Concurrency and threshold throttling for [Sidekiq][sidekiq].
 
@@ -145,7 +146,7 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
 
 ## Copyright
 
-Copyright (c) 2015 SensorTower Inc.
+Copyright (c) 2015-2016 SensorTower Inc.
 See LICENSE.md for further details.
 
 

data/lib/sidekiq/throttled/fetch.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq"
-require "sidekiq/throttled/unit_of_work"
+require "sidekiq/throttled/fetch/unit_of_work"
 require "sidekiq/throttled/queues_pauser"
 require "sidekiq/throttled/queue_name"
 
@@ -22,6 +22,8 @@ module Sidekiq
         @queues.uniq! if @strict
       end
 
+      # Retrieves job from redis.
+      #
       # @return [Sidekiq::Throttled::UnitOfWork, nil]
       def retrieve_work
         work = brpop
@@ -30,21 +32,36 @@ module Sidekiq
         work = UnitOfWork.new(*work)
         return work unless Throttled.throttled? work.job
 
-        Sidekiq.redis do |conn|
-          conn.lpush(QueueName.expand(work.queue_name), work.job)
-        end
+        work.throttled_requeue
 
         nil
       end
 
+      class << self
+        # Requeues all given units as a single operation.
+        #
+        # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
+        # @param [Array<Fetch::UnitOfWork>] units
+        # @return [void]
+        def bulk_requeue(units, _options)
+          return if units.empty?
+
+          Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+          Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
+          Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
+        rescue => e
+          Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
+        end
+      end
+
       private
 
-      # Tries to pop pair of `queue` and job `message` out of sidekiq queue.
+      # Tries to pop pair of `queue` and job `message` out of sidekiq queues.
       #
       # @see http://redis.io/commands/brpop
       # @return [Array(String, String), nil]
       def brpop
-        queues = build_queues_list
+        queues = filter_queues(@strict ? @queues : @queues.shuffle.uniq)
 
         if queues.empty?
           sleep TIMEOUT
@@ -57,9 +74,10 @@ module Sidekiq
       # Returns list of queues to try to fetch jobs from.
       #
       # @note It may return an empty array.
+      # @param [Array<String>] queues
       # @return [Array<String>]
-      def build_queues_list
-        QueuesPauser.instance.filter(@strict ? @queues : @queues.shuffle.uniq)
+      def filter_queues(queues)
+        QueuesPauser.instance.filter(queues)
       end
     end
   end

data/lib/sidekiq/throttled/fetch/unit_of_work.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+require "sidekiq/throttled/queue_name"
+
+module Sidekiq
+  module Throttled
+    class Fetch
+      # BRPOP response envelope.
+      #
+      # @see Throttled::Fetch
+      # @private
+      class UnitOfWork
+        # @return [String] Redis key where job was pulled from
+        attr_reader :queue
+
+        # @return [String] Job's JSON payload
+        attr_reader :job
+
+        # @param [String] queue Redis key where job was pulled from
+        # @param [String] job Job's JSON payload
+        def initialize(queue, job)
+          @queue = queue
+          @job   = job
+        end
+
+        # Callback that is called by `Sidekiq::Processor` when job was
+        # succeccfully processed. Most this is used by `ReliableFetch`
+        # of Sidekiq Pro/Enterprise to remove job from running queue.
+        #
+        # @return [void]
+        def acknowledge
+          # do nothing
+        end
+
+        # Normalized `queue` name.
+        #
+        # @see QueueName.normalize
+        # @return [String]
+        def queue_name
+          @queue_name ||= QueueName.normalize queue
+        end
+
+        # Pushes job back to the tail of the queue, so that it will be popped
+        # first next time fetcher will pull job.
+        #
+        # @note This is triggered when job was not finished and Sidekiq server
+        #   process was terminated. It is a reverse of whatever fetcher was
+        #   doing to pull the job out of queue.
+        #
+        # @return [void]
+        def requeue
+          Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
+        end
+
+        # Pushes job back to the head of the queue, so that job won't be tried
+        # immediately after it was requeued (in most cases).
+        #
+        # @note This is triggered when job is throttled. So it is same operation
+        #   Sidekiq performs upon `Sidekiq::Worker.perform_async` call.
+        #
+        # @return [void]
+        def throttled_requeue
+          Sidekiq.redis { |conn| conn.lpush(QueueName.expand(queue_name), job) }
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/throttled/version.rb

@@ -3,6 +3,6 @@
 module Sidekiq
   module Throttled
     # Gem version
-    VERSION = "0.6.1".freeze
+    VERSION = "0.6.2".freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-throttled
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Alexey V Zapparov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-30 00:00:00.000000000 Z
+date: 2016-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -66,6 +66,7 @@ files:
 - lib/sidekiq/throttled/communicator/listener.rb
 - lib/sidekiq/throttled/errors.rb
 - lib/sidekiq/throttled/fetch.rb
+- lib/sidekiq/throttled/fetch/unit_of_work.rb
 - lib/sidekiq/throttled/middleware.rb
 - lib/sidekiq/throttled/queue_name.rb
 - lib/sidekiq/throttled/queues_pauser.rb
@@ -77,7 +78,6 @@ files:
 - lib/sidekiq/throttled/strategy/threshold.lua
 - lib/sidekiq/throttled/strategy/threshold.rb
 - lib/sidekiq/throttled/testing.rb
-- lib/sidekiq/throttled/unit_of_work.rb
 - lib/sidekiq/throttled/version.rb
 - lib/sidekiq/throttled/web.rb
 - lib/sidekiq/throttled/web/index.html.erb

data/lib/sidekiq/throttled/unit_of_work.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq"
-
-require "sidekiq/throttled/queue_name"
-
-module Sidekiq
-  module Throttled
-    # BRPOP response envelope.
-    #
-    # @see Throttled::Fetch
-    # @private
-    class UnitOfWork
-      # @return [String] Redis key where job was pulled from
-      attr_reader :queue
-
-      # @return [String] Job's JSON payload
-      attr_reader :job
-
-      # @param [String] queue Redis key where job was pulled from
-      # @param [String] job Job's JSON payload
-      def initialize(queue, job)
-        @queue = queue
-        @job   = job
-      end
-
-      # Callback that is called by `Sidekiq::Processor` when job was
-      # succeccfully processed. Most this is used by `ReliableFetch`
-      # of Sidekiq Pro/Enterprise to remove job from running queue.
-      #
-      # @return [void]
-      def acknowledge
-        # do nothing
-      end
-
-      # Normalized `queue` name.
-      #
-      # @see QueueName.normalize
-      # @return [String]
-      def queue_name
-        @queue_name ||= QueueName.normalize queue
-      end
-
-      # Pushes job back to the queue.
-      #
-      # @note This is triggered when job was not finished and Sidekiq server
-      #   process was terminated (shutdowned). Thus it should be reverse of
-      #   whatever fetcher was doing to pull the job out of queue.
-      #
-      # @return [void]
-      def requeue
-        Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
-      end
-    end
-  end
-end