sidekiq-fast-enq 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 40869777a1a6d37a3e5eab0b5f020934a49aef19
+  data.tar.gz: 18d7f23470bed6ff4c82dbd63b3f6476bec77fea
+SHA512:
+  metadata.gz: 39c51cd9e46ec73bda022e0486fb18383012833d6427b8772af8c091e412ff0618788488a20f0771184be3d0419b2940585c3fc951f11a4f94be7addfdef8ede
+  data.tar.gz: c0d00501fc81d12b4bbf4e3bb8a839ac4e0a2b55a85d1f6ba8de7237e4f3a20b733b9b6df64b4986344394ea3dd13d639d03380a76c545890d34a38786a10b12

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/MIT_LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 WHI, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,23 @@
+This gem provides a much more efficien implementation for checking the Sidekiq scheduled and retry queues. This can provide a significant performance boost for large sidekiq implementations that utilize many processes. It can also reduce load on the redis server.
+
+### TL;DR
+
+The default implementation included with the sidekiq gem works very well when there are only a few processes running. However, with a large number of processes it is vulnerable to race conditions when there are a lot of scheduled jobs and a lot of sidekiq processes. Each process will maintain a thread that checks the scheduled and retry job queues. These queues are kept in sorted sets with the timestamp to run the jobs as the sorting key. Each process will run a redis command to sort the set and return the first element whose key is less than the current timestamp. It then removes the job from the scheduled queue and adds it to the appropriate job queue. The problem is that if there are a lot schedule jobs, the set sorting operation can take a not insignificant amount of time. If there are lot of processes this will lead to a lot of race conditions where for each job run the large set will be sorted many times.
+
+This gem re-implements the same logic, but using a server side Lua script so that the sorting and popping from the list become an atomic operation eliminating the race conditions as well as using more efficient redis commands.
+
+On a single sidekiq process this implementation is about twice as fast. With 64 processes it's about nine times as fast and puts significantly less load on redis.
+
+This plugin does not alter any sidekiq internal code or data structures.
+
+### Usage
+
+In your sidekiq configuration you need to set the `:scheduled_enq` option to `SidekiqFastEnq` (only available in sidekiq 3.4.0 and later). You might also want to hard code a value for the `:poll_interval_average` option as well. If this option is not set the polling interval for checking the scheduled queues is based on the number of processes in an effort to reduce the effects of the race condition. It is not needed with this code and scheduled jobs will be enqueued closer to their scheduled time without it.
+
+```ruby
+Sidekiq.options[:scheduled_enq] = SidekiqFastEnq
+Sidekiq.options[:poll_interval_average] = 30
+```
+
+Since this gem eliminates the race condition of having too many processes processing th
+Note: this gem utilizes server side Lua scripting so you must be using Redis Server 2.6.0 or later.

data/Rakefile

@@ -0,0 +1,57 @@
+require "bundler/gem_tasks"
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'RVM likes to call it tests'
+task :tests => :test
+
+begin
+  require 'rspec'
+  require 'rspec/core/rake_task'
+  desc 'Run the unit tests'
+  RSpec::Core::RakeTask.new(:test)
+rescue LoadError
+  task :test do
+    STDERR.puts "You must have rspec 2.0 installed to run the tests"
+  end
+end
+
+task :load_test, [:jobs_size, :workers, :fast] do |t, args|
+  require 'celluloid'
+  require File.expand_path('../lib/sidekiq-fast-enq', __FILE__)
+  require 'sidekiq/scheduled'
+  require 'sidekiq/api'
+  
+  class FastEnqLoadTestWorker
+    include Sidekiq::Worker
+    def perform()
+    end
+  end
+  
+  jobs_size = args[:jobs_size].to_i
+  workers_size = args[:workers].to_i
+  klass = (args[:fast] == 'fast' ? SidekiqFastEnq : Sidekiq::Scheduled::Enq)
+  
+  Sidekiq.configure_server do |config|
+    config.redis = {:namespace => "sidekiq_fast_enq_load_test"}
+  end
+  
+  Sidekiq::ScheduledSet.new.clear
+  jobs_size.times do
+    FastEnqLoadTestWorker.perform_in(rand)
+  end
+  
+  t = Time.now
+  workers_size.times do
+    fork do
+      klass.new.enqueue_jobs
+    end
+  end
+  
+  workers_size.times do
+    Process.wait
+  end
+  
+  puts "Enqueued #{jobs_size} jobs in #{Time.now - t} seconds"
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/sidekiq-fast-enq.rb

@@ -0,0 +1,121 @@
+require 'sidekiq'
+
+# Implementation of the Sidekiq::Scheduled::Enq class that uses a server side Lua script
+# to atomically get the next scheduled job to run and then pops it from the list. This
+# works much better in large sidekiq deployments with many processes because it eliminates
+# race conditions checking the scheduled queues.
+class SidekiqFastEnq
+  DEFAULT_BATCH_SIZE = 1000
+  
+  def initialize(batch_size = nil)
+    batch_size ||= (Sidekiq.options[:fast_enq_batch_size] || DEFAULT_BATCH_SIZE)
+    @script = lua_script(batch_size)
+    Sidekiq.redis do |conn|
+      @script_sha_1 = conn.script(:load, @script)
+    end
+  end
+  
+  def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = nil)
+    sorted_sets ||= Sidekiq::Scheduled::SETS
+    logger = Sidekiq::Logging.logger
+    
+    # A job's "score" in Redis is the time at which it should be processed.
+    # Just check Redis for the set of jobs with a timestamp before now.
+    Sidekiq.redis do |conn|
+      namespace = conn.namespace if conn.respond_to?(:namespace)
+      sorted_sets.each do |sorted_set|
+        redis_set = (namespace ? "#{namespace}:#{sorted_set}" : sorted_set)
+        jobs_count = 0
+        start_time = Time.now
+        pop_time = 0.0
+        enqueue_time = 0.0
+        
+        # Get the next item in the queue if it's score (time to execute) is <= now.
+        # We need to go through the list one at a time to reduce the risk of something
+        # going wrong between the time jobs are popped from the scheduled queue and when
+        # they are pushed onto a work queue and losing the jobs.
+        loop do
+          t = Time.now
+          job = pop_job(conn, redis_set, now)
+          pop_time += (Time.now - t)
+          break if job.nil?
+          t = Time.now
+          Sidekiq::Client.push(Sidekiq.load_json(job))
+          enqueue_time += (Time.now - t)
+          jobs_count += 1
+          logger.debug("enqueued #{sorted_set}: #{job}") if logger && logger.debug?
+        end
+        
+        if jobs_count > 0 && logger && logger.info?
+          loop_time = Time.now - start_time
+          logger.info("SidekiqFastEnq enqueued #{jobs_count} from #{sorted_set} in #{loop_time.round(3)}s (pop: #{pop_time.round(3)}s; enqueue: #{enqueue_time.round(3)}s)")
+        end
+      end
+    end
+  end
+
+  private
+
+  # Invoke a Lua script on the server to pop the next job from a sorted set that should have
+  # been run before "now".
+  def pop_job(conn, sorted_set, now)
+    eval_script(conn, @script, @script_sha_1, [sorted_set, now])
+  end
+  
+  # Evaluate and execute a Lua script on the redis server.
+  def eval_script(conn, script, sha1, argv=[])
+    begin
+      conn.evalsha(sha1, [], argv)
+    rescue Redis::CommandError => e
+      if e.message.include?('NOSCRIPT'.freeze)
+        t = Time.now
+        sha1 = conn.script(:load, script)
+        Sidekiq::Logging.logger.info("loaded script #{sha1} in #{Time.now - t}s")
+        retry
+      else
+        raise e
+      end
+    end
+  end
+
+  # Lua script that will atomically get the next element from the sorted set of scheduled jobs
+  # and pop it from the list.
+  def lua_script(batch_size)
+    batch_size = batch_size.to_i
+    batch_size = DEFAULT_BATCH_SIZE if batch_size <= 0
+    <<-LUA
+    local sorted_set = ARGV[1]
+    local now = tonumber(ARGV[2])
+    local ready_cache = sorted_set .. '.cache'
+    
+    while true do
+      -- Check a cached list of jobs that are ready to execute
+      local job = redis.call('lpop', ready_cache)
+      if not job then
+        -- If no jobs in the cache then get the next 100 jobs ready to be executed
+        local ready_jobs = redis.call('zrangebyscore', sorted_set, '-inf', now, 'LIMIT', 0, #{batch_size})
+        if #ready_jobs == 1 then
+          job = ready_jobs[1]
+        elseif #ready_jobs > 1 then
+          -- If more than one job is ready, throw them in the cache which is faster to access than the sorted set
+          redis.call('rpush', ready_cache, unpack(ready_jobs))
+          -- Set an expiration on the cache since it's just a cache. The sorted set is still the canonical list.
+          redis.call('expire', ready_cache, 60)
+          job = redis.call('lpop', ready_cache)
+        end
+      end
+
+      if job then
+        -- Verify that the job was still in the sorted set when we remove. Could happen if
+        -- another sidekiq process is still using the standard Enq mechanism.
+        local removed = redis.call('zrem', sorted_set, job)
+        if removed > 0 then
+          return job
+        end
+      else
+        return nil
+      end
+    end
+    LUA
+  end
+end

data/sidekiq_fast_enq.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "sidekiq-fast-enq"
+  spec.version       = File.read(File.expand_path("../VERSION", __FILE__)).chomp
+  spec.authors       = ["We Heart It", "Brian Durand"]
+  spec.email         = ["dev@weheartit.com", "bbdurand@gmail.com"]
+  spec.summary       = "More efficient scheduled job queue implementation for sidekiq"
+  spec.description   = "More efficient scheduled job queue implementation for sidekiq to increase throughput in large installations."
+  spec.homepage      = "https://github.com/weheartit/sidekiq_fast_enq"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency('sidekiq', '~>3.4')
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "timecop"
+end

data/spec/sidekiq-fast-enq_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe SidekiqFastEnq do
+  
+  let(:scheduled_set){ Sidekiq::ScheduledSet.new }
+  let(:retry_set){ Sidekiq::RetrySet.new }
+  let(:default_queue){ Sidekiq::Queue.new }
+  
+  before :each do
+    scheduled_set.clear
+    retry_set.clear
+    default_queue.clear
+  end
+  
+  it "should return without doing anything if there are no scheduled jobs" do
+    SidekiqFastEnq.new.enqueue_jobs
+    expect(scheduled_set.size).to eq(0)
+    expect(retry_set.size).to eq(0)
+    expect(default_queue.size).to eq(0)
+  end
+  
+  it "should enqueue a single elligible job from the scheduled jobs queue" do
+    Timecop.travel(Time.now - 3600){ FastEnqTestWorker.perform_in(60, 'one') }
+    SidekiqFastEnq.new.enqueue_jobs
+    expect(scheduled_set.size).to eq(0)
+    expect(retry_set.size).to eq(0)
+    expect(default_queue.size).to eq(1)
+  end
+  
+  it "should enqueue all elligible jobs from the scheduled jobs queue" do
+    Timecop.travel(Time.now - 3600){ FastEnqTestWorker.perform_in(60, 'one') }
+    Timecop.travel(Time.now - 3600){ FastEnqTestWorker.perform_in(900, 'two') }
+    FastEnqTestWorker.perform_in(10, 'three')
+    SidekiqFastEnq.new.enqueue_jobs
+    expect(scheduled_set.size).to eq(1)
+    expect(retry_set.size).to eq(0)
+    expect(default_queue.size).to eq(2)
+  end
+  
+  it "should enqueue all elligible jobs from the scheduled jobs queue when there are a lot of them" do
+    Timecop.travel(Time.now - 3600) do
+      200.times do
+        FastEnqTestWorker.perform_in(60, 'one')
+      end
+    end
+    FastEnqTestWorker.perform_in(10, 'three')
+    SidekiqFastEnq.new.enqueue_jobs
+    expect(scheduled_set.size).to eq(1)
+    expect(retry_set.size).to eq(0)
+    expect(default_queue.size).to eq(200)
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,35 @@
+# Breaks if not required. Sidekiq doesn't directly require in
+# the load process.
+
+sidekiq_version = Array(ENV["SIDEKIQ_VERSION"] || "~>3.0")
+gem 'sidekiq', *sidekiq_version
+
+require File.expand_path('../../lib/sidekiq-fast-enq', __FILE__)
+require 'timecop'
+require 'celluloid'
+require 'sidekiq/scheduled'
+require 'sidekiq/api'
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+  
+  Sidekiq.configure_server do |config|
+    config.redis = {:namespace => "sidekiq_fast_enq_test"}
+  end
+  Sidekiq.options[:scheduled_enq] = SidekiqFastEnq
+  Sidekiq::Logging.logger = Logger.new(StringIO.new)
+end
+
+class FastEnqTestWorker
+  include Sidekiq::Worker
+  
+  def perform(arg)
+  end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-fast-enq
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- We Heart It
+- Brian Durand
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: More efficient scheduled job queue implementation for sidekiq to increase
+  throughput in large installations.
+email:
+- dev@weheartit.com
+- bbdurand@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- MIT_LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- lib/sidekiq-fast-enq.rb
+- sidekiq_fast_enq.gemspec
+- spec/sidekiq-fast-enq_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/weheartit/sidekiq_fast_enq
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: More efficient scheduled job queue implementation for sidekiq
+test_files:
+- spec/sidekiq-fast-enq_spec.rb
+- spec/spec_helper.rb