sidekiq 4.0.0.pre2 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ae613def4b402732162cb005ca3179a1a810e09
-  data.tar.gz: 36aab3dd08ed5370fce1f2a6de7fae6973222dfb
+  metadata.gz: 154fd4d987e5f7c32fad42a20176d27e958bd49f
+  data.tar.gz: a33449473a2210e1f79e3f814c50f6fe982ff101
 SHA512:
-  metadata.gz: a2b0c7ab9631689ea1afd0bdbb0d5eb0122ac4258b245c91c7ffed51b3d65dfb3c1c5b5492c235f8ab0ef628b5a154e2a9cab4a9cb75dc2e1f2029670f63d223
-  data.tar.gz: 1d1391df672310f61563ba6a5125602104690a587da72a58038f33492626ab80cad60a7bba61507cadbf26d0bcc9d864d863adb43592e6543965072c5240af86
+  metadata.gz: 264d2c5a210a978ca8ce9f2a78962cc4f4d62966444788b7d942644881baec212133123f3434ffafdcc21d5bd9c0130eef007195a945f86771a33da5b2d1fbea
+  data.tar.gz: 6e651a3d38dd4518a3db01abd5febdb9a4c739d16873f2ce3ffa4998bd5f5a2285303437cc6f489cb43d032590bf741006d50170470c333ed31cfa62c1168178

data/4.0-Upgrade.md

@@ -25,6 +25,16 @@ gem 'redis-namespace'
   `concurrency + 2` connections in your pool or Sidekiq will exit.
   When in doubt, let Sidekiq size the connection pool for you.
 
+* There's a new testing API based off the `Sidekiq::Queues` namespace. All
+  assertions made against the Worker class still work as expected.
+```ruby
+assert_equal 0, Sidekiq::Queues["default"].size
+HardWorker.perform_async("log")
+assert_equal 1, Sidekiq::Queues["default"].size
+assert_equal "log", Sidekiq::Queues["default"].first['args'][0]
+Sidekiq::Queues.clear_all
+```
+
 ## Upgrade
 
 First, make sure you are using Redis 2.8 or greater. Next:

data/Changes.md

@@ -7,6 +7,16 @@
   and to remove dependencies.  This has resulted in major speedups, as
   [detailed on my blog](http://www.mikeperham.com/2015/10/14/optimizing-sidekiq/).
 - See the [4.0 upgrade notes](4.0-Upgrade.md) for more detail.
+- There's a new testing API based off the `Sidekiq::Queues` namespace. All
+  assertions made against the Worker class still work as expected.
+  [#2659, brandonhilkert]
+```ruby
+assert_equal 0, Sidekiq::Queues["default"].size
+HardWorker.perform_async("log")
+assert_equal 1, Sidekiq::Queues["default"].size
+assert_equal "log", Sidekiq::Queues["default"].first['args'][0]
+Sidekiq::Queues.clear_all
+```
 
 3.5.3
 -----------

data/Ent-Changes.md

@@ -3,9 +3,23 @@ Sidekiq Enterprise Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+1.0.0.pre1
+----------
+
+- Enterprise 1.x targets Sidekiq 4.x.
+- Rewrite several features to remove Celluloid dependency.  No
+  functional changes.
+
+0.7.8
+----------
+
+- Fix `unique_for: false` [#2658]
+
+
 0.7.7
 ----------
 
+- Enterprise 0.x targets Sidekiq 3.x.
 - Fix racy shutdown event which could lead to disappearing periodic
   jobs, requires Sidekiq >= 3.5.3.
 - Add new :leader event which is fired when a process gains leadership.

data/Pro-Changes.md

@@ -3,11 +3,22 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
-3.0.0
+3.0.0.pre2
+-----------
+
+- Fix excessive connection usage by reliable fetch.
+
+3.0.0.pre1
 -----------
 
 - See the [Pro 3.0 release notes](Pro-3.0-Upgrade.md).
 
+2.1.3
+-----------
+
+- Don't enable strict priority if using weighted queueing like `-q a,1 -q b,1`
+- Safer JSON mangling in Lua [#2639]
+
 2.1.2
 -----------
 

data/bin/sidekiqctl

@@ -41,9 +41,13 @@ class Sidekiqctl
   end
 
   def fetch_process
-    Process.getpgid(pid)
+    Process.kill(0, pid)
   rescue Errno::ESRCH
     done "Process doesn't exist", :error
+  # We were not allowed to send a signal, but the process must have existed
+  # when Process.kill() was called.
+  rescue Errno::EPERM
+    return pid
   end
 
   def done(msg, error = nil)
@@ -67,10 +71,12 @@ class Sidekiqctl
     `kill -TERM #{pid}`
     kill_timeout.times do
       begin
-        Process.getpgid(pid)
+        Process.kill(0, pid)
       rescue Errno::ESRCH
         FileUtils.rm_f pidfile
         done 'Sidekiq shut down gracefully.'
+      rescue Errno::EPERM
+        done 'Not permitted to shut down Sidekiq.'
       end
       sleep 1
     end

data/bin/sidekiqload

@@ -5,6 +5,7 @@
 $TESTING = false
 
 #require 'ruby-prof'
+Bundler.require(:default)
 
 require_relative '../lib/sidekiq/cli'
 require_relative '../lib/sidekiq/launcher'
@@ -30,6 +31,7 @@ Sidekiq.configure_server do |config|
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
+  config.reliable! if defined?(Sidekiq::Pro)
 end
 
 class LoadWorker
@@ -140,6 +142,7 @@ end
 begin
   #RubyProf::exclude_threads = [ Monitoring ]
   #RubyProf.start
+  fire_event(:startup)
   Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
   Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)

data/lib/sidekiq/cli.rb

@@ -69,7 +69,7 @@ module Sidekiq
         # touch the connection pool so it is created before we
         # fire startup and start multithreading.
         ver = conn.info['redis_version']
-        raise "Oops, you are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+        raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
       end
 
       # Before this point, the process is initializing with just the main thread.

data/lib/sidekiq/testing.rb

@@ -68,15 +68,15 @@ module Sidekiq
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+          Queues.jobs[job['queue']] << Sidekiq.load_json(Sidekiq.dump_json(job))
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          job['jid'] ||= SecureRandom.hex(12)
           klass = job['class'].constantize
-          klass.jobs.unshift Sidekiq.load_json(Sidekiq.dump_json(job))
-          klass.perform_one
+          job['id'] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
         end
         true
       else
@@ -85,6 +85,64 @@ module Sidekiq
     end
   end
 
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # The data is structured as a hash with queue name as hash key and array
+    # of job data as the value.
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs[queue.to_s]
+      end
+
+      def jobs
+        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def clear_all
+        jobs.clear
+      end
+    end
+  end
+
   module Worker
     ##
     # The Sidekiq testing infrastructure overrides perform_async
@@ -143,28 +201,36 @@ module Sidekiq
     #
     module ClassMethods
 
+      # Queue for this worker
+      def queue
+        self.sidekiq_options["queue"].to_s
+      end
+
       # Jobs queued for this worker
       def jobs
-        Worker.jobs[self]
+        Queues.jobs[queue].select { |job| job["class"] == self.to_s }
       end
 
       # Clear all jobs for this worker
       def clear
-        jobs.clear
+        Queues.jobs[queue].clear
       end
 
       # Drain and run all jobs for this worker
       def drain
-        while job = jobs.shift do
-          process_job(job)
+        while jobs.any?
+          next_job = jobs.first
+          Queues.jobs[queue].delete_if { |job| job["jid"] == next_job["jid"] }
+          process_job(next_job)
         end
       end
 
       # Pop out a single job and perform it
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
-        job = jobs.shift
-        process_job(job)
+        next_job = jobs.first
+        Queues.jobs[queue].delete_if { |job| job["jid"] == next_job["jid"] }
+        process_job(next_job)
       end
 
       def process_job(job)
@@ -183,18 +249,22 @@ module Sidekiq
 
     class << self
       def jobs # :nodoc:
-        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+        Queues.jobs.values.flatten
       end
 
       # Clear all queued jobs across all workers
       def clear_all
-        jobs.clear
+        Queues.clear_all
       end
 
       # Drain all queued jobs across all workers
       def drain_all
-        until jobs.values.all?(&:empty?) do
-          jobs.keys.each(&:drain)
+        while jobs.any?
+          worker_classes = jobs.map { |job| job["class"] }.uniq
+
+          worker_classes.each do |worker_class|
+            worker_class.constantize.drain
+          end
         end
       end
     end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "4.0.0.pre2"
+  VERSION = "4.0.0"
 end

data/lib/sidekiq/web_helpers.rb

@@ -160,7 +160,7 @@ module Sidekiq
       options = options.stringify_keys
       params.merge(options).map do |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{value}" : next
-      end.join("&")
+      end.compact.join("&")
     end
 
     def truncate(text, truncate_after_chars = 2000)
@@ -245,5 +245,12 @@ module Sidekiq
     def product_version
       "Sidekiq v#{Sidekiq::VERSION}"
     end
+
+    def redis_connection_and_namespace
+      @redis_connection_and_namespace ||= begin
+        namespace_suffix = namespace == nil ? '' : "##{namespace}"
+        "#{redis_connection}#{namespace_suffix}"
+      end
+    end
   end
 end

data/sidekiq.gemspec

@@ -16,10 +16,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
   gem.add_dependency                  'redis', '~> 3.2', '>= 3.2.1'
-  gem.add_dependency                  'redis-namespace', '~> 1.5', '>= 1.5.2'
   gem.add_dependency                  'connection_pool', '~> 2.2', '>= 2.2.0'
   gem.add_dependency                  'json', '~> 1.0'
-  gem.add_dependency                  'concurrent-ruby', '1.0.0.pre4'
+  gem.add_dependency                  'concurrent-ruby', '~> 1.0'
+  gem.add_development_dependency      'redis-namespace', '~> 1.5', '>= 1.5.2'
   gem.add_development_dependency      'sinatra', '~> 1.4', '>= 1.4.6'
   gem.add_development_dependency      'minitest', '~> 5.7', '>= 5.7.0'
   gem.add_development_dependency      'rake', '~> 10.0'

data/test/test_testing_fake.rb

@@ -54,6 +54,7 @@ class TestTesting < Sidekiq::Test
 
     after do
       Sidekiq::Testing.disable!
+      Sidekiq::Queues.clear_all
     end
 
     it 'stubs the async call' do
@@ -93,7 +94,7 @@ class TestTesting < Sidekiq::Test
     it 'stubs the enqueue_to call' do
       assert_equal 0, EnqueuedWorker.jobs.size
       assert Sidekiq::Client.enqueue_to('someq', EnqueuedWorker, 1, 2)
-      assert_equal 1, EnqueuedWorker.jobs.size
+      assert_equal 1, Sidekiq::Queues['someq'].size
     end
 
     it 'executes all stored jobs' do
@@ -263,6 +264,68 @@ class TestTesting < Sidekiq::Test
     it 'can execute a job' do
       DirectWorker.execute_job(DirectWorker.new, [2, 3])
     end
+  end
+
+  describe 'queue testing' do
+    before do
+      require 'sidekiq/testing'
+      Sidekiq::Testing.fake!
+    end
+
+    after do
+      Sidekiq::Testing.disable!
+      Sidekiq::Queues.clear_all
+    end
 
+    class QueueWorker
+      include Sidekiq::Worker
+      def perform(a, b)
+        a + b
+      end
+    end
+
+    class AltQueueWorker
+      include Sidekiq::Worker
+      sidekiq_options queue: :alt
+      def perform(a, b)
+        a + b
+      end
+    end
+
+    it 'finds enqueued jobs' do
+      assert_equal 0, Sidekiq::Queues["default"].size
+
+      QueueWorker.perform_async(1, 2)
+      QueueWorker.perform_async(1, 2)
+      AltQueueWorker.perform_async(1, 2)
+
+      assert_equal 2, Sidekiq::Queues["default"].size
+      assert_equal [1, 2], Sidekiq::Queues["default"].first["args"]
+
+      assert_equal 1, Sidekiq::Queues["alt"].size
+    end
+
+    it 'clears out all queues' do
+      assert_equal 0, Sidekiq::Queues["default"].size
+
+      QueueWorker.perform_async(1, 2)
+      QueueWorker.perform_async(1, 2)
+      AltQueueWorker.perform_async(1, 2)
+
+      Sidekiq::Queues.clear_all
+
+      assert_equal 0, Sidekiq::Queues["default"].size
+      assert_equal 0, Sidekiq::Queues["alt"].size
+    end
+
+    it 'finds jobs enqueued by client' do
+      Sidekiq::Client.push(
+        'class' => 'NonExistentWorker',
+        'queue' => 'missing',
+        'args' => [1]
+      )
+
+      assert_equal 1, Sidekiq::Queues["missing"].size
+    end
   end
 end

data/web/assets/stylesheets/application.css

@@ -748,3 +748,7 @@ div.interval-slider input {
     max-width: 350px;
   }
 }
+
+.redis-url {
+  text-overflow: ellipsis;
+}

data/web/views/_footer.erb

@@ -6,16 +6,11 @@
             <p class="navbar-text" style="color:white;"><%= product_version %></p>
           </li>
           <li>
-            <p class="navbar-text redis-url">Redis: <%= redis_connection %></p>
+            <p class="navbar-text redis-url" title="<%= redis_connection_and_namespace %>"><%= redis_connection_and_namespace %></p>
           </li>
           <li>
-            <p class="navbar-text"><%= t('Time') %>: <%= Time.now.utc.strftime('%H:%M:%S UTC') %></p>
+            <p class="navbar-text"><%= Time.now.utc.strftime('%H:%M:%S UTC') %></p>
           </li>
-          <% if namespace %>
-              <li>
-                <p class="navbar-text redis-namespace"><%= t('Namespace') %>: <%= namespace %></p>
-              </li>
-          <% end %>
         </ul>
     </div>
   </div>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 4.0.0.pre2
+  version: 4.0.0
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-02 00:00:00.000000000 Z
+date: 2015-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -31,47 +31,41 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.2.1
 - !ruby/object:Gem::Dependency
-  name: redis-namespace
+  name: connection_pool
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: 2.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
-  name: connection_pool
+  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -85,19 +79,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: redis-namespace
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.pre4
-  type: :runtime
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0.pre4
+        version: 1.5.2
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
@@ -376,9 +376,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.5