resque-fifo-queue 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b962de3ad6dec018f3c39ae6dd0a1bce29552c6a
+  data.tar.gz: 7bb016d06b063bd45459668ede1de61df13b0d6d
+SHA512:
+  metadata.gz: 3026f465971b56dd870ca6f177d0783ec0070d9231b0e4b17657664cadfe6bf22d753bbeeb7b64392472fddfa75cd8d828ed5b9bf675bfe8f0a1637f708ea764
+  data.tar.gz: c584f50175d9ae9e9ef934427e0b10688f40867ab40a913f0616ec909974e21baf2275e9fe956fff3ce951d893636ae5cbc17641056350b0b265416e9bf8f221

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+*.gem
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+before_install: gem install bundler -v 1.14.6

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at joseph.dayo@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in resque-fifo-queue.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Joseph Emmanuel Dayo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+# Resque::Fifo::Queue
+
+Implementation of a sharded First-in First-out queue using Redis and Resque.
+
+This gem unables you to guarantee in-order job processing based on a shard key. Useful for business requirements that are race-condition prone or needs something processed in a streaming manner (jobs that require preservation of chronological order).
+
+Sharding is automatically managed depending on the number of workers available. Durability is guaranteed with failover resharding using a consitent hash. Built on the reliability of resque and is pure ruby which simplifies deployment if you are already using resque with ruby on rails.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'resque-fifo-queue'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install resque-fifo-queue
+
+## Usage
+
+This adds a new task, to run fifo queues:
+
+```bash
+rake resque:fifo-worker
+rake resque:fifo-workers
+```
+
+Available options are similar to rake resque:work
+
+Workers will assign their own queue names which is automatically managed by the sharding algorithm.
+
+Supports the same parameters as resque:work but creates additional queues behind the scenes
+to support fifo queues.
+
+fifo workers can also double duty as standard resque worker in order to simplify resource sharing:
+
+```bash
+QUEUE=high rake resque:fifo-worker
+```
+
+Aside from being a worker that processes jobs from the fifo queue it will process jobs from the high queue as well.
+
+Sample Usage
+------------
+
+To start a job using a fifo strategy:
+
+```ruby
+class SampleJob
+  def self.perform(*args)
+    # run your resque job here
+  end
+end
+
+shard_key = "user_00001"
+
+# These async jobs will be guaranteed to run one after another in a single worker
+
+Resque::Plugins::Fifo::Queue::Manager.enqueue_to(shard_key, SampleJob, "hello")
+Resque::Plugins::Fifo::Queue::Manager.enqueue_to(shard_key, SampleJob, "hello1")
+
+```
+
+## Resque web extensions
+
+This gem adds a FIFO_queue tab under resque web where you can see information about
+workers and queues used. This can be accessed via:
+
+http://localhost:3000/resque/fifo_queue
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Ensuring workers are updated
+
+Since only one worker is assigned to a particular shard, it is important to make sure a worker is running. The
+fifo shard table is updated everytime a worker is started or stopped properly (using kill -S QUITE).
+
+Though for some cases a scheduled task is necessary to make sure the worker list is constantly updated:
+
+```yaml
+# resque_schedule.yml
+auto_refresh_fifo_queues:
+  cron: '*/5 * * * * UTC'
+  class: Resque::Plugins::Fifo::Queue::DrainWorker
+  queue: fifo_refresh
+  description: 'Check if fifo workers are still valid and update worker table'
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/resque-fifo-queue. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require 'resque/tasks'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "resque/fifo/queue"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/init.rb

@@ -0,0 +1 @@
+require 'resque/fifo/queue'

data/lib/resque/fifo/constants.rb

@@ -0,0 +1,11 @@
+
+module Resque
+  module Plugins
+    module Fifo
+      WORKER_QUEUE_NAMESPACE = "fifo-managed-queue"
+
+      module Queue
+      end
+    end
+  end
+end

data/lib/resque/fifo/queue.rb

@@ -0,0 +1,12 @@
+require 'resque'
+require 'resque/fifo/constants'
+require "redis"
+require "redlock"
+require 'xxhash'
+require 'resque_solo'
+require "resque/plugins/fifo/queue/version"
+require "resque/plugins/fifo/server"
+require "resque/plugins/fifo/extensions"
+require "resque/plugins/fifo/queue/drain_worker"
+require "resque/plugins/fifo/queue/manager"
+require 'resque/plugins/fifo/worker'

data/lib/resque/fifo/tasks.rb

@@ -0,0 +1,25 @@
+require 'resque/fifo/queue'
+
+task "resque:fifo-worker" => :environment do
+  prefix = ENV['PREFIX'] || 'fifo'
+  worker = Resque::Plugins::Fifo::Worker.new
+  worker.prepare
+  worker.log "Starting worker #{self}"
+  worker.work(ENV['INTERVAL'] || 5) # interval, will block
+end
+
+task "resque:fifo-workers" => :environment do
+  threads = []
+
+  if ENV['COUNT'].to_i < 1
+    abort "set COUNT env var, e.g. $ COUNT=2 rake resque:workers"
+  end
+
+  ENV['COUNT'].to_i.times do
+    threads << Thread.new do
+      system "rake resque:fifo-worker"
+    end
+  end
+
+  threads.each { |thread| thread.join }
+end

data/lib/resque/plugins/fifo/extensions.rb

@@ -0,0 +1,7 @@
+Resque.send(:define_singleton_method, :queues) do
+  data_store.queue_names.reject { |name| name.start_with?(Resque::Plugins::Fifo::WORKER_QUEUE_NAMESPACE) }
+end
+
+Resque.send(:define_singleton_method, :all_queues) do
+  data_store.queue_names
+end

data/lib/resque/plugins/fifo/queue/drain_worker.rb

@@ -0,0 +1,15 @@
+module Resque
+  module Plugins
+    module Fifo
+      module Queue
+        class DrainWorker
+          include Resque::Plugins::UniqueJob
+
+          def self.perform
+            Resque::Plugins::Fifo::Queue::Manager.new.update_workers
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/fifo/queue/manager.rb

@@ -0,0 +1,429 @@
+require 'set'
+
+module Resque
+  module Plugins
+    module Fifo
+      module Queue
+        class Manager
+          DLM_TTL = 30000
+          attr_accessor :queue_prefix
+
+          def initialize(queue_prefix = 'fifo')
+            @queue_prefix = queue_prefix
+          end
+
+          def fifo_hash_table_name
+            "fifo-queue-lookup-#{@queue_prefix}"
+          end
+
+          def queue_prefix
+            "#{Resque::Plugins::Fifo::WORKER_QUEUE_NAMESPACE}-#{@queue_prefix}"
+          end
+
+          def pending_queue_name
+            "#{queue_prefix}-pending"
+          end
+
+          def compute_queue_name(key)
+            index = compute_index(key)
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+
+            return pending_queue_name if slots.empty?
+
+            slots.reverse.each do |slot|
+              slice, queue = slot.split('#')
+              if index > slice.to_i
+                return queue
+              end
+            end
+
+            _slice, queue_name = slots.last.split('#')
+
+            queue_name
+          end
+
+          def enqueue(key, klass, *args)
+            queue = compute_queue_name(key)
+
+            redis_client.incr "queue-stats-#{queue}"
+            Resque.validate(klass, queue)
+            if Resque.inline? && inline?
+              # Instantiating a Resque::Job and calling perform on it so callbacks run
+              # decode(encode(args)) to ensure that args are normalized in the same manner as a non-inline job
+              Resque::Job.new(:inline, {'class' => klass, 'args' => Resque.decode(Resque.encode(args)), 'fifo_key' => key, 'enqueue_ts' => 0}).perform
+            else
+              Resque.push(queue, :class => klass.to_s, :args => args, fifo_key: key, :enqueue_ts => Time.now.to_i)
+            end
+          end
+
+          # method for stubbing in tests
+          def inline?
+            Resque.inline?
+          end
+
+          def clear_stats
+            redis_client.del "fifo-stats-max-delay"
+            redis_client.del "fifo-stats-accumulated-delay"
+            redis_client.del "fifo-stats-accumulated-count"
+            redis_client.del "fifo-stats-dht-rehash"
+            redis_client.del "fifo-stats-accumulated-recalc-time"
+            redis_client.del "fifo-stats-accumulated-recalc-count"
+
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+            slots.each_with_index.collect do |slot, index|
+              slice, queue = slot.split('#')
+              redis_client.del "queue-stats-#{queue}"
+            end
+          end
+
+          def get_stats_max_delay
+            redis_client.get("fifo-stats-max-delay") || 0
+          end
+
+          def get_stats_avg_dht_recalc
+            accumulated_delay = redis_client.get("fifo-stats-accumulated-recalc-time") || 0
+            total_items = redis_client.get("fifo-stats-accumulated-recalc-count") || 0
+            return 0 if total_items == 0
+
+            return accumulated_delay.to_f / total_items.to_f
+          end
+
+          def get_stats_avg_delay
+            accumulated_delay = redis_client.get("fifo-stats-accumulated-delay") || 0
+            total_items = redis_client.get("fifo-stats-accumulated-count") || 0
+            return 0 if total_items == 0
+
+            return accumulated_delay.to_f / total_items.to_f
+          end
+
+          def dht_times_rehashed
+            redis_client.get("fifo-stats-dht-rehash") || 0
+          end
+
+          def all_stats
+            {
+              dht_times_rehashed: dht_times_rehashed,
+              avg_delay: get_stats_avg_delay,
+              avg_dht_recalc: get_stats_avg_dht_recalc,
+              max_delay: get_stats_max_delay
+            }
+          end
+
+          def self.enqueue_to(key, klass, *args)
+            enqueue_topic('fifo', key, klass, *args)
+          end
+
+          def self.enqueue_topic(topic, key, klass, *args)
+            # Perform before_enqueue hooks. Don't perform enqueue if any hook returns false
+            before_hooks = Plugin.before_enqueue_hooks(klass).collect do |hook|
+              klass.send(hook, *args)
+            end
+
+            return nil if before_hooks.any? { |result| result == false }
+
+            manager = Resque::Plugins::Fifo::Queue::Manager.new(topic)
+            manager.enqueue(key, klass, *args)
+
+            Plugin.after_enqueue_hooks(klass).each do |hook|
+              klass.send(hook, *args)
+            end
+
+            return true
+          end
+
+          def dump_dht
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+            slots.each_with_index.collect do |slot, index|
+              slice, queue = slot.split('#')
+              [slice.to_i, queue]
+            end
+          end
+
+          def pretty_dump
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+            slots.each_with_index.collect do |slot, index|
+              slice, queue = slot.split('#')
+              puts "Slice  ##{slice} -> #{queue}"
+            end
+          end
+
+          def peek_pending
+            Resque.peek(pending_queue_name, 0, 0)
+          end
+
+          def pending_total
+            redis_client.llen "queue:#{pending_queue_name}"
+          end
+
+          def dump_queue_names
+            dump_dht.collect { |item| item[1] }
+          end
+
+          def worker_for_queue(queue_name)
+            Resque.workers.collect do |worker|
+              w_queue_name = worker.queues.select { |name| name.start_with?("#{queue_prefix}-") }.first
+              return worker if w_queue_name == queue_name
+            end.compact
+            nil
+          end
+
+          def dump_queues
+            query_available_queues.collect do |queue|
+              [queue, Resque.peek(queue,0,0)]
+            end.to_h
+          end
+
+          def pretty_dump_queues
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+            slots.each_with_index.collect do |slot, index|
+              slice, queue = slot.split('#')
+              puts "#Slice #{slice}"
+
+              puts "#{Resque.peek(queue,0,0).to_s.gsub('},',"},\n")},"
+              puts "\n"
+            end
+          end
+
+          def dump_queues_with_slices
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+            slots.collect do |slot, index|
+              slice, queue = slot.split('#')
+              worker = worker_for_queue(queue)
+
+              hostname = '?'
+              status = '?'
+              pid = '?'
+              started = '?'
+              heartbeat = '?'
+
+              if worker
+                hostname = worker.hostname
+                status = worker.paused? ? 'paused' : worker.state.to_s
+                pid = worker.pid
+                started = worker.started
+                heartbeat = worker.heartbeat
+              end
+
+              [slice, queue, hostname, pid, status, started, heartbeat, get_processed_count(queue), Resque.peek(queue,0,0).size ]
+            end
+          end
+
+          def get_processed_count(queue)
+            redis_client.get("queue-stats-#{queue}") || 0
+          end
+
+          def dump_queues_sorted
+            queues = dump_queues
+            dht = dump_dht.collect do |item|
+              _slice, queue_name = item
+              queues[queue_name]
+            end
+          end
+
+          def update_workers
+            # query removed workers
+            start_time = Time.now.to_i
+            redlock.lock("fifo_queue_lock-#{queue_prefix}", DLM_TTL) do |locked|
+              if locked
+                start_timestamp = redis_client.get "fifo_update_timestamp-#{queue_prefix}"
+
+                process_dht
+                cleanup_queues
+                log("reinserting items from pending")
+                reinsert_pending_items(pending_queue_name)
+
+                # check if something tried to request an update, if so we requie again
+                current_timestamp = redis_client.get "fifo_update_timestamp-#{queue_prefix}"
+
+                if start_timestamp != current_timestamp
+                  request_refresh
+                end
+              else
+                log("unable to lock DHT.")
+              end
+            end
+
+            end_time = Time.now.to_i
+
+            redis_client.set("fifo-stats-accumulated-recalc-time", end_time - start_time)
+            redis_client.incr "fifo-stats-accumulated-recalc-count"
+          end
+
+          def request_refresh
+            if Resque.inline?
+              # Instantiating a Resque::Job and calling perform on it so callbacks run
+              # decode(encode(args)) to ensure that args are normalized in the same manner as a non-inline job
+              Resque::Job.new(:inline, {'class' => Resque::Plugins::Fifo::Queue::DrainWorker, 'args' => []}).perform
+            else
+              redis_client.set "fifo_update_timestamp-#{queue_prefix}", Time.now.to_s
+              Resque.push(:fifo_refresh, :class => Resque::Plugins::Fifo::Queue::DrainWorker.to_s, :args => [])
+            end
+
+          end
+
+          def orphaned_queues
+            current_queues = dump_queue_names
+            Resque.all_queues.reject do |queue|
+              !queue.start_with?(queue_prefix) || current_queues.include?(queue)
+            end
+          end
+
+          private
+
+          def cleanup_queues
+            current_queues = dump_queue_names
+            Resque.all_queues.each do |queue|
+              if queue.start_with?(queue_prefix)
+                next if current_queues.include?(queue)
+
+                if redis_client.llen("queue:#{queue}") > 0
+                  log("transfer non empty orphaned queue items to pending")
+                  transfer_queues(queue, pending_queue_name)
+                end
+
+                log("remove orphaned queue #{queue}.")
+                Resque.remove_queue(queue)
+              end
+            end
+          end
+
+          def process_dht
+            slots = redis_client.lrange fifo_hash_table_name, 0, -1
+
+            current_queues = slots.map { |slot| slot.split('#')[1] }.uniq
+
+            available_queues = query_available_queues
+            # no change don't update
+            return if available_queues.sort == current_queues.sort
+
+            redis_client.incr "fifo-stats-dht-rehash"
+
+            remove_list = slots.select do |slot|
+              _slice, queue = slot.split('#')
+              !available_queues.include?(queue)
+            end
+
+            remove_list.each do |slot|
+              _slice, queue = slot.split('#')
+              log "queue #{queue} removed."
+              redis_client.lrem fifo_hash_table_name, -1, slot
+              transfer_queues(queue, pending_queue_name)
+              redis_client.del "queue-stats-#{queue}"
+            end
+
+            added_queues = available_queues.each do |queue|
+              if !current_queues.include?(queue)
+                insert_slot(queue)
+                log "queue #{queue} was added."
+              end
+            end
+          end
+
+          def log(message)
+            puts message
+          end
+
+          def insert_slot(queue)
+            new_slice =  generate_new_slice # generate random 32-bit integer
+            insert_queue_to_slice new_slice, queue
+          end
+
+          def generate_new_slice
+            XXhash.xxh32(rand(0..2**32).to_s)
+          end
+
+          def insert_queue_to_slice(slice, queue)
+            queue_str = "#{slice}##{queue}"
+            log "insert #{queue} -> #{slice}"
+            slots = redis_client.lrange(fifo_hash_table_name, 0, -1)
+
+            if slots.empty?
+              redis_client.rpush(fifo_hash_table_name, queue_str)
+              return
+            end
+
+            _b_slice, prev_queue = slots.last.split('#')
+            slots.each do |slot|
+              slot_slice, s_queue = slot.split('#')
+              if slice < slot_slice.to_i
+                redlock.lock!("queue_lock-#{prev_queue}", DLM_TTL) do |_lock_info|
+                  pause_queues([prev_queue]) do
+                    redis_client.linsert(fifo_hash_table_name, 'BEFORE', slot, queue_str)
+                    transfer_queues(prev_queue, pending_queue_name)
+                  end
+                end
+                return
+              end
+
+              prev_queue = s_queue
+            end
+
+            _slot_slice, s_queue = slots.last.split('#')
+            pause_queues([s_queue]) do
+              transfer_queues(s_queue, pending_queue_name)
+              redis_client.rpush(fifo_hash_table_name, queue_str)
+            end
+          end
+
+          def reinsert_pending_items(from_queue)
+            redis_client.llen("queue:#{from_queue}").times do
+              slot = redis_client.lpop "queue:#{from_queue}"
+              queue_json = JSON.parse(slot)
+              target_queue = compute_queue_name(queue_json['fifo_key'])
+              log "#{queue_json['fifo_key']}: #{from_queue} -> #{target_queue}"
+              redis_client.rpush("queue:#{target_queue}", slot)
+            end
+          end
+
+          def pause_queues(queue_names = [], &block)
+            begin
+              queue_names.each do |queue_name|
+                worker = worker_for_queue(queue_name)
+                worker.pause_processing if worker
+              end
+
+              block.()
+            ensure
+              queue_names.each do |queue_name|
+                worker = worker_for_queue(queue_name)
+                worker.unpause_processing if worker
+              end
+            end
+          end
+
+          def transfer_queues(from_queue, to_queue)
+            log "transfer: #{from_queue} -> #{to_queue}"
+            redis_client.llen("queue:#{from_queue}").times do
+              redis_client.rpoplpush("queue:#{from_queue}", "queue:#{to_queue}")
+            end
+          end
+
+          def redis_client
+            Resque.redis
+          end
+
+          def redlock
+              Redlock::Client.new [redis_client.redis], {
+              retry_count:   30,
+              retry_delay:   1000, # milliseconds
+              retry_jitter:  100,  # milliseconds
+              redis_timeout: 1  # seconds
+            }
+          end
+
+          def compute_index(key)
+            XXhash.xxh32(key)
+          end
+
+          def query_available_queues
+            expired_workers = Resque::Worker.all_workers_with_expired_heartbeats
+
+            Resque.workers.reject { |w| expired_workers.include?(w) }.collect do |worker|
+              worker.queues.select { |name| name.start_with?("#{queue_prefix}-") }.first
+            end.compact
+          end
+        end
+      end
+    end
+  end
+end