sidekiq-fast-enq 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 42316bf316539012e9cd71544606ad581c47917d
-  data.tar.gz: 9817a10809a564cec6ab8e87612a009c2a7ed8f7
+SHA256:
+  metadata.gz: 4c2ef4bb287f73b4dc43e32b604d16e98dc284822f399952e13cee7df1869836
+  data.tar.gz: b9c4efb591b505b94733ba6722b50c99ddd0a6151da9b54ab5d16534364cda15
 SHA512:
-  metadata.gz: 3eb9ce115d05af9f31166b3752bbc3bd5fd8260dcfce625328bcb5be6306261f766a1f5f773559a192c267af9069d9dde2f9f0ec6943bbd16cc3821a93777d40
-  data.tar.gz: ec68435cbeb752ead5db389d518dba994af9b6f3f757c4693b620e2f668008deee51e1a5e88ca57a584312f5c7f2ada729f4b90705b0c7e8cd133b2dddb26286
+  metadata.gz: d1ed8261dd525b1de1820f676b9c545a0cc6486cef1a3ed8e078ea9eec7ecf2e67f94aaba11d347b5d59f2a8d8956612d564560182f7d3652fe9ec00a167dcc7
+  data.tar.gz: dc9c18fc20a0a0ff75bacf4428c94f4eb5c8665824f1060755508a1273d05bb7889c73499cff4bd5a60c24b01360c5d5a7a52754a991a40996af39d9bfb29cd0

data/CHANGELOG.md

@@ -0,0 +1,29 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 1.0.3
+
+### Added
+
+- Compatibility with Sidekiq 7
+
+## 1.0.2
+
+### Added
+
+- Compatibility with Sidekiq 6
+
+## 1.0.2
+
+### Changed
+
+- Update gemspec to specify compatibility with sidekiq 4
+
+# 1.0.0
+
+### Added
+
+- Initial release

data/MIT_LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2015 WHI, Inc.
+Copyright (c) 2015 Brian Durand
 
 MIT License
 

data/README.md

@@ -1,3 +1,9 @@
+Sidekiq Fast Enqueuing
+
+[![Continuous Integration](https://github.com/bdurand/sidekiq-fast-enq/actions/workflows/continuous_integration.yml/badge.svg)](https://github.com/bdurand/sidekiq-fast-enq/actions/workflows/continuous_integration.yml)
+[![Regression Test](https://github.com/bdurand/sidekiq-fast-enq/actions/workflows/regression_test.yml/badge.svg)](https://github.com/bdurand/sidekiq-fast-enq/actions/workflows/regression_test.yml)
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard)
+
 This gem provides a much more efficien implementation for checking the Sidekiq scheduled and retry queues. This can provide a significant performance boost for large sidekiq implementations that utilize many processes. It can also reduce load on the redis server.
 
 ### TL;DR
@@ -14,10 +20,47 @@ This plugin does not alter any sidekiq internal code or data structures.
 
 In your sidekiq configuration you need to set the `:scheduled_enq` option to `SidekiqFastEnq` (only available in sidekiq 3.4.0 and later). You might also want to hard code a value for the `:poll_interval_average` option as well. If this option is not set the polling interval for checking the scheduled queues is based on the number of processes in an effort to reduce the effects of the race condition. It is not needed with this code and scheduled jobs will be enqueued closer to their scheduled time without it.
 
+```ruby
+Sidekiq.default_configuration[:scheduled_enq] = SidekiqFastEnq
+Sidekiq.default_configuration[:poll_interval_average] = 30
+```
+
+For Sidekiq versions prior to version 7:
+
 ```ruby
 Sidekiq.options[:scheduled_enq] = SidekiqFastEnq
 Sidekiq.options[:poll_interval_average] = 30
 ```
 
-Since this gem eliminates the race condition of having too many processes processing th
-Note: this gem utilizes server side Lua scripting so you must be using Redis Server 2.6.0 or later.
+
+### Redis requirement
+
+Redis server 2.6 or greater is required for this code.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sidekiq-fast-enq'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install sidekiq-fast-enq
+```
+
+## Contributing
+
+Fork the repository and open a pull request on GitHub.
+
+Please use the [standardrb](https://github.com/testdouble/standard) syntax and lint your code with `standardrb --fix` before submitting.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.0.3

data/lib/sidekiq-fast-enq.rb

@@ -1,4 +1,6 @@
-require 'sidekiq'
+# frozen_string_literal: true
+
+require "sidekiq"
 
 # Implementation of the Sidekiq::Scheduled::Enq class that uses a server side Lua script
 # to atomically get the next scheduled job to run and then pops it from the list. This
@@ -6,19 +8,20 @@ require 'sidekiq'
 # race conditions checking the scheduled queues.
 class SidekiqFastEnq
   DEFAULT_BATCH_SIZE = 1000
-  
+
   def initialize(batch_size = nil)
-    batch_size ||= (Sidekiq.options[:fast_enq_batch_size] || DEFAULT_BATCH_SIZE)
+    options = (Sidekiq.respond_to?(:default_configuration) ? Sidekiq.default_configuration : Sidekiq.options)
+    batch_size ||= (options[:fast_enq_batch_size] || DEFAULT_BATCH_SIZE)
     @script = lua_script(batch_size)
     Sidekiq.redis do |conn|
       @script_sha_1 = conn.script(:load, @script)
     end
   end
-  
+
   def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = nil)
     sorted_sets ||= Sidekiq::Scheduled::SETS
-    logger = Sidekiq::Logging.logger
-    
+    logger = Sidekiq.logger
+
     # A job's "score" in Redis is the time at which it should be processed.
     # Just check Redis for the set of jobs with a timestamp before now.
     Sidekiq.redis do |conn|
@@ -29,7 +32,7 @@ class SidekiqFastEnq
         start_time = Time.now
         pop_time = 0.0
         enqueue_time = 0.0
-        
+
         # Get the next item in the queue if it's score (time to execute) is <= now.
         # We need to go through the list one at a time to reduce the risk of something
         # going wrong between the time jobs are popped from the scheduled queue and when
@@ -43,10 +46,10 @@ class SidekiqFastEnq
           Sidekiq::Client.push(Sidekiq.load_json(job))
           enqueue_time += (Time.now - t)
           jobs_count += 1
-          logger.debug("enqueued #{sorted_set}: #{job}") if logger && logger.debug?
+          logger.debug("enqueued #{sorted_set}: #{job}") if logger&.debug?
         end
-        
-        if jobs_count > 0 && logger && logger.info?
+
+        if jobs_count > 0 && logger&.info?
           loop_time = Time.now - start_time
           logger.info("SidekiqFastEnq enqueued #{jobs_count} from #{sorted_set} in #{loop_time.round(3)}s (pop: #{pop_time.round(3)}s; enqueue: #{enqueue_time.round(3)}s)")
         end
@@ -61,20 +64,18 @@ class SidekiqFastEnq
   def pop_job(conn, sorted_set, now)
     eval_script(conn, @script, @script_sha_1, [sorted_set, now])
   end
-  
+
   # Evaluate and execute a Lua script on the redis server.
-  def eval_script(conn, script, sha1, argv=[])
-    begin
-      conn.evalsha(sha1, [], argv)
-    rescue Redis::CommandError => e
-      if e.message.include?('NOSCRIPT'.freeze)
-        t = Time.now
-        sha1 = conn.script(:load, script)
-        Sidekiq::Logging.logger.info("loaded script #{sha1} in #{Time.now - t}s")
-        retry
-      else
-        raise e
-      end
+  def eval_script(conn, script, sha1, argv = [])
+    conn.evalsha(sha1, [], argv)
+  rescue Redis::CommandError => e
+    if e.message.include?("NOSCRIPT")
+      t = Time.now
+      sha1 = conn.script(:load, script)
+      Sidekiq::Logging.logger.info("loaded script #{sha1} in #{Time.now - t}s")
+      retry
+    else
+      raise e
     end
   end
 
@@ -87,7 +88,7 @@ class SidekiqFastEnq
     local sorted_set = ARGV[1]
     local now = tonumber(ARGV[2])
     local ready_cache = sorted_set .. '.cache'
-    
+
     while true do
       -- Check a cached list of jobs that are ready to execute
       local job = redis.call('lpop', ready_cache)

data/sidekiq_fast_enq.gemspec

@@ -1,26 +1,34 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
 Gem::Specification.new do |spec|
-  spec.name          = "sidekiq-fast-enq"
-  spec.version       = File.read(File.expand_path("../VERSION", __FILE__)).chomp
-  spec.authors       = ["We Heart It", "Brian Durand"]
-  spec.email         = ["dev@weheartit.com", "bbdurand@gmail.com"]
-  spec.summary       = "More efficient scheduled job queue implementation for sidekiq"
-  spec.description   = "More efficient scheduled job queue implementation for sidekiq to increase throughput in large installations."
-  spec.homepage      = "https://github.com/weheartit/sidekiq_fast_enq"
-  spec.license       = "MIT"
+  spec.name = "sidekiq-fast-enq"
+  spec.version = File.read(File.expand_path("VERSION", __dir__)).strip
+  spec.authors = ["Brian Durand"]
+  spec.email = ["bbdurand@gmail.com"]
+
+  spec.summary = "More efficient scheduled job queue implementation for sidekiq to increase throughput in large installations."
+  spec.homepage = "https://github.com/bdurand/sidekiq-fast-enq"
+  spec.license = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  ignore_files = %w[
+    .
+    Appraisals
+    Gemfile
+    Gemfile.lock
+    Rakefile
+    bin/
+    gemfiles/
+    spec/
+  ]
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| ignore_files.any? { |path| f.start_with?(path) } }
+  end
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency('sidekiq', '>=3.4')
+  spec.add_dependency("sidekiq", ">=5.0")
+
+  spec.add_development_dependency "bundler"
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "timecop"
+  spec.required_ruby_version = ">= 2.5"
 end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-fast-enq
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.3
 platform: ruby
 authors:
-- We Heart It
 - Brian Durand
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-19 00:00:00.000000000 Z
+date: 2023-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -17,44 +16,16 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,44 +38,24 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: More efficient scheduled job queue implementation for sidekiq to increase
-  throughput in large installations.
+description:
 email:
-- dev@weheartit.com
 - bbdurand@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- HISTORY.txt
+- CHANGELOG.md
 - MIT_LICENSE.txt
 - README.md
-- Rakefile
 - VERSION
 - lib/sidekiq-fast-enq.rb
 - sidekiq_fast_enq.gemspec
-- spec/sidekiq-fast-enq_spec.rb
-- spec/spec_helper.rb
-homepage: https://github.com/weheartit/sidekiq_fast_enq
+homepage: https://github.com/bdurand/sidekiq-fast-enq
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -112,18 +63,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5
-signing_key: 
+rubygems_version: 3.4.12
+signing_key:
 specification_version: 4
-summary: More efficient scheduled job queue implementation for sidekiq
-test_files:
-- spec/sidekiq-fast-enq_spec.rb
-- spec/spec_helper.rb
+summary: More efficient scheduled job queue implementation for sidekiq to increase
+  throughput in large installations.
+test_files: []

data/.gitignore

@@ -1,17 +0,0 @@
-*.gem
-*.rbc
-.bundle
-.config
-.yardoc
-Gemfile.lock
-InstalledFiles
-_yardoc
-coverage
-doc/
-lib/bundler/man
-pkg
-rdoc
-spec/reports
-test/tmp
-test/version_tmp
-tmp

data/HISTORY.txt

@@ -1,7 +0,0 @@
-1.0.1
-
-Update gemspec to specify compatibility with sidekiq 4.
-
-1.0.0
-
-Initial release.

data/Rakefile

@@ -1,57 +0,0 @@
-require "bundler/gem_tasks"
-
-desc 'Default: run unit tests.'
-task :default => :test
-
-desc 'RVM likes to call it tests'
-task :tests => :test
-
-begin
-  require 'rspec'
-  require 'rspec/core/rake_task'
-  desc 'Run the unit tests'
-  RSpec::Core::RakeTask.new(:test)
-rescue LoadError
-  task :test do
-    STDERR.puts "You must have rspec 2.0 installed to run the tests"
-  end
-end
-
-task :load_test, [:jobs_size, :workers, :fast] do |t, args|
-  require 'celluloid'
-  require File.expand_path('../lib/sidekiq-fast-enq', __FILE__)
-  require 'sidekiq/scheduled'
-  require 'sidekiq/api'
-  
-  class FastEnqLoadTestWorker
-    include Sidekiq::Worker
-    def perform()
-    end
-  end
-  
-  jobs_size = args[:jobs_size].to_i
-  workers_size = args[:workers].to_i
-  klass = (args[:fast] == 'fast' ? SidekiqFastEnq : Sidekiq::Scheduled::Enq)
-  
-  Sidekiq.configure_server do |config|
-    config.redis = {:namespace => "sidekiq_fast_enq_load_test"}
-  end
-  
-  Sidekiq::ScheduledSet.new.clear
-  jobs_size.times do
-    FastEnqLoadTestWorker.perform_in(rand)
-  end
-  
-  t = Time.now
-  workers_size.times do
-    fork do
-      klass.new.enqueue_jobs
-    end
-  end
-  
-  workers_size.times do
-    Process.wait
-  end
-  
-  puts "Enqueued #{jobs_size} jobs in #{Time.now - t} seconds"
-end

data/spec/sidekiq-fast-enq_spec.rb

@@ -1,53 +0,0 @@
-require 'spec_helper'
-
-describe SidekiqFastEnq do
-  
-  let(:scheduled_set){ Sidekiq::ScheduledSet.new }
-  let(:retry_set){ Sidekiq::RetrySet.new }
-  let(:default_queue){ Sidekiq::Queue.new }
-  
-  before :each do
-    scheduled_set.clear
-    retry_set.clear
-    default_queue.clear
-  end
-  
-  it "should return without doing anything if there are no scheduled jobs" do
-    SidekiqFastEnq.new.enqueue_jobs
-    expect(scheduled_set.size).to eq(0)
-    expect(retry_set.size).to eq(0)
-    expect(default_queue.size).to eq(0)
-  end
-  
-  it "should enqueue a single elligible job from the scheduled jobs queue" do
-    Timecop.travel(Time.now - 3600){ FastEnqTestWorker.perform_in(60, 'one') }
-    SidekiqFastEnq.new.enqueue_jobs
-    expect(scheduled_set.size).to eq(0)
-    expect(retry_set.size).to eq(0)
-    expect(default_queue.size).to eq(1)
-  end
-  
-  it "should enqueue all elligible jobs from the scheduled jobs queue" do
-    Timecop.travel(Time.now - 3600){ FastEnqTestWorker.perform_in(60, 'one') }
-    Timecop.travel(Time.now - 3600){ FastEnqTestWorker.perform_in(900, 'two') }
-    FastEnqTestWorker.perform_in(10, 'three')
-    SidekiqFastEnq.new.enqueue_jobs
-    expect(scheduled_set.size).to eq(1)
-    expect(retry_set.size).to eq(0)
-    expect(default_queue.size).to eq(2)
-  end
-  
-  it "should enqueue all elligible jobs from the scheduled jobs queue when there are a lot of them" do
-    Timecop.travel(Time.now - 3600) do
-      200.times do
-        FastEnqTestWorker.perform_in(60, 'one')
-      end
-    end
-    FastEnqTestWorker.perform_in(10, 'three')
-    SidekiqFastEnq.new.enqueue_jobs
-    expect(scheduled_set.size).to eq(1)
-    expect(retry_set.size).to eq(0)
-    expect(default_queue.size).to eq(200)
-  end
-  
-end

data/spec/spec_helper.rb

@@ -1,36 +0,0 @@
-# Breaks if not required. Sidekiq doesn't directly require in
-# the load process.
-
-sidekiq_version = Array(ENV["SIDEKIQ_VERSION"] || ">=3.4")
-gem 'sidekiq', *sidekiq_version
-
-require File.expand_path('../../lib/sidekiq-fast-enq', __FILE__)
-require 'timecop'
-require 'sidekiq/version'
-require 'celluloid' if Sidekiq::VERSION.to_i < 4
-require 'sidekiq/scheduled'
-require 'sidekiq/api'
-
-RSpec.configure do |config|
-  config.run_all_when_everything_filtered = true
-  config.filter_run :focus
-
-  # Run specs in random order to surface order dependencies. If you find an
-  # order dependency and want to debug it, you can fix the order by providing
-  # the seed, which is printed after each run.
-  #     --seed 1234
-  config.order = 'random'
-  
-  Sidekiq.configure_server do |config|
-    config.redis = {:namespace => "sidekiq_fast_enq_test"}
-  end
-  Sidekiq.options[:scheduled_enq] = SidekiqFastEnq
-  Sidekiq::Logging.logger = Logger.new(StringIO.new)
-end
-
-class FastEnqTestWorker
-  include Sidekiq::Worker
-  
-  def perform(arg)
-  end
-end