sidekiq 3.2.2 → 3.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac4be785d484b8190a4aeb09fc93d21f4897f2f1
-  data.tar.gz: 9955d71f3356939e2cc06f7287f66d850b18604c
+  metadata.gz: 56f90dbb4ffba7f12619e992cdd36e92c0c26c90
+  data.tar.gz: c5a9d6faf0ae1d1aa6b2ecb65b2adc282032a5aa
 SHA512:
-  metadata.gz: ba6263b604d50445ad7582a49870daf4af2105570d0d43fe6702128903bdda651c640435eff1d4f61fdeb828193ed91fc6552b536f87fc8cee3d4b313b7c13f9
-  data.tar.gz: 260002fc92ba5a50aa910402f388ca1f77f2c21aab076fcd59f8e9d04c68e48cbd2b2889f89ef1caee745e9a585af86f1ea9caa65c46af68983d23f66ec0e55c
+  metadata.gz: 730a5cd70e49741c0426ee7f7d467939f72adae0ea276c00cc45cf331e48794fe8c40d1d84c4432c0d92c20bf7f795c06be775c2cae26e98170967673922c302
+  data.tar.gz: 6261ef3999c2a4ad2674172ce1c84d4bef1fb172c0952994aa0232f943e7671add7c72984f381b3c4a7198a723f72cc3362abc6a81baee136e162992ee63f3b7

data/Changes.md

@@ -1,6 +1,18 @@
+3.2.3
+-----------
+
+- Clean invalid bytes from error message before converting to JSON (requires Ruby 2.1+) [#1705]
+- Add queues list for each process to the Busy page. [davetoxa, #1897]
+- Fix for crash caused by empty config file. [jordan0day, #1901]
+- Add Rails Worker generator, `rails g sidekiq:worker User` will create `app/workers/user_worker.rb`. [seuros, #1909]
+- Fix Web UI rendering with huge job arguments [jhass, #1918]
+- Minor refactoring of Sidekiq::Client internals, for Sidekiq Pro. [#1919]
+
 3.2.2
 -----------
 
+- **This version of Sidekiq will no longer start on Ruby 1.9.**  Sidekiq
+  3 does not support MRI 1.9 but we've allowed it to run before now.
 - Fix issue which could cause Sidekiq workers to disappear from the Busy
   tab while still being active [#1884]
 - Add "Back to App" button in Web UI.  You can set the button link via

data/Pro-Changes.md

@@ -3,10 +3,25 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/pro](http://sidekiq.org/pro) for more details and how to buy.
 
+1.8.0
+-----------
+
+- Fix long standing Batch race condition where Batches can complete
+  before they have been fully defined or only half-defined. Requires
+  Sidekiq 3.2.3. [#1919]
+
+
+1.7.6
+-----------
+
+- Quick release to verify #1919
+
+
 1.7.5
 -----------
 
 - Fix job filtering within the Dead tab.
+- Add APIs and wiki documentation for invalidating jobs within a batch.
 
 
 1.7.4

data/lib/generators/sidekiq/templates/worker.rb.erb

@@ -0,0 +1,9 @@
+<% module_namespacing do -%>
+class <%= class_name %>Worker
+  include Sidekiq::Worker
+
+  def perform(*args)
+    # Do something
+  end
+end
+<% end -%>

data/lib/generators/sidekiq/templates/worker_spec.rb.erb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+<% module_namespacing do -%>
+RSpec.describe <%= class_name %>Worker, :type => :worker do
+    pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/generators/sidekiq/templates/worker_test.rb.erb

@@ -0,0 +1,8 @@
+require_relative 'test_helper'
+<% module_namespacing do -%>
+class <%= class_name %>WorkerTest < MiniTest::Unit::TestCase
+    def test_example
+      skip "add some examples to (or delete) #{__FILE__}"
+    end
+end
+<% end -%>

data/lib/generators/sidekiq/worker_generator.rb

@@ -0,0 +1,49 @@
+require 'rails/generators/named_base'
+
+module Sidekiq
+  module Generators # :nodoc:
+    class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
+      desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
+
+      check_class_collision suffix: 'Worker'
+
+      def self.default_generator_root
+        File.dirname(__FILE__)
+      end
+
+      def create_worker_file
+        template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
+      end
+
+      def create_test_file
+        if defined?(RSpec)
+          create_worker_spec
+        else
+          create_worker_test
+        end
+      end
+
+      private
+
+      def create_worker_spec
+        template_file = File.join(
+            'spec/workers',
+            class_path,
+            "#{file_name}_spec.rb"
+        )
+        template 'worker_spec.rb.erb', template_file
+      end
+
+      def create_worker_test
+        template_file = File.join(
+            'test/workers',
+            class_path,
+            "#{file_name}_test.rb"
+        )
+        template 'worker_test.rb.erb', template_file
+      end
+
+
+    end
+  end
+end

data/lib/sidekiq/api.rb

@@ -326,7 +326,7 @@ module Sidekiq
         Sidekiq.redis do |conn|
           conn.multi do
             conn.zadd('dead', now, message)
-            conn.zremrangebyscore('dead', '-inf', now - DeadSet::DEAD_JOB_TIMEOUT)
+            conn.zremrangebyscore('dead', '-inf', now - DeadSet::TIMEOUT)
             conn.zremrangebyrank('dead', 0, - DeadSet::MAX_JOBS)
           end
         end
@@ -520,7 +520,7 @@ module Sidekiq
   # Allows enumeration of dead jobs within Sidekiq.
   #
   class DeadSet < JobSet
-    DEAD_JOB_TIMEOUT = 180 * 24 * 60 * 60 # 6 months
+    TIMEOUT = 180 * 24 * 60 * 60 # 6 months
     MAX_JOBS = 10_000
 
     def initialize

data/lib/sidekiq/cli.rb

@@ -353,7 +353,7 @@ module Sidekiq
     def parse_config(cfile)
       opts = {}
       if File.exist?(cfile)
-        opts = YAML.load(ERB.new(IO.read(cfile)).result)
+        opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
         opts = opts.merge(opts.delete(environment) || {})
         parse_queues(opts, opts.delete(:queues) || [])
       else

data/lib/sidekiq/client.rb

@@ -176,23 +176,26 @@ module Sidekiq
     private
 
     def raw_push(payloads)
-      pushed = false
       @redis_pool.with do |conn|
-        if payloads.first['at']
-          pushed = conn.zadd('schedule', payloads.map do |hash|
-            at = hash.delete('at').to_s
-            [at, Sidekiq.dump_json(hash)]
-          end)
-        else
-          q = payloads.first['queue']
-          to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
-          _, pushed = conn.multi do
-            conn.sadd('queues', q)
-            conn.lpush("queue:#{q}", to_push)
-          end
+        conn.multi do
+          atomic_push(conn, payloads)
         end
       end
-      pushed
+      true
+    end
+
+    def atomic_push(conn, payloads)
+      if payloads.first['at']
+        conn.zadd('schedule', payloads.map do |hash|
+          at = hash.delete('at').to_s
+          [at, Sidekiq.dump_json(hash)]
+        end)
+      else
+        q = payloads.first['queue']
+        to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
+        conn.sadd('queues', q)
+        conn.lpush("queue:#{q}", to_push)
+      end
     end
 
     def process_single(worker_class, item)

data/lib/sidekiq/launcher.rb

@@ -49,7 +49,7 @@ module Sidekiq
         poller.terminate if poller.alive?
 
         manager.async.stop(:shutdown => true, :timeout => @options[:timeout])
-        manager.wait(:shutdown)
+        manager.wait(:shutdown) if manager.alive?
 
         # Requeue everything in case there was a worker who grabbed work while stopped
         Sidekiq::Fetcher.strategy.bulk_requeue([], @options)

data/lib/sidekiq/middleware/i18n.rb

@@ -1,3 +1,9 @@
+#
+# Simple middleware to save the current locale and restore it when the job executes.
+# Use it by requiring it in your initializer:
+#
+#     require 'sidekiq/middleware/i18n'
+#
 module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -1,4 +1,5 @@
 require 'sidekiq/scheduled'
+require 'sidekiq/api'
 
 module Sidekiq
   module Middleware
@@ -86,7 +87,16 @@ module Sidekiq
           else
             queue
           end
-          msg['error_message'] = e.message[0..10_000]
+
+          # App code can stuff all sorts of crazy binary data into the error message
+          # that won't convert to JSON.
+          m = e.message[0..10_000]
+          if m.respond_to?(:scrub!)
+            m.force_encoding("utf-8")
+            m.scrub!
+          end
+
+          msg['error_message'] = m
           msg['error_class'] = e.class.name
           count = if msg['retry_count']
             msg['retried_at'] = Time.now.to_f
@@ -122,9 +132,6 @@ module Sidekiq
 
         private
 
-        DEAD_JOB_TIMEOUT = 180 * 24 * 60 * 60 # 6 months
-        MAX_JOBS = 10_000
-
         def retries_exhausted(worker, msg)
           logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
           begin
@@ -145,7 +152,7 @@ module Sidekiq
           Sidekiq.redis do |conn|
             conn.multi do
               conn.zadd('dead', now, payload)
-              conn.zremrangebyscore('dead', '-inf', now - DeadSet::DEAD_JOB_TIMEOUT)
+              conn.zremrangebyscore('dead', '-inf', now - DeadSet::TIMEOUT)
               conn.zremrangebyrank('dead', 0, -DeadSet::MAX_JOBS)
             end
           end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.2.2"
+  VERSION = "3.2.3"
 end

data/lib/sidekiq/web.rb

@@ -221,6 +221,7 @@ module Sidekiq
           enqueued:   sidekiq_stats.enqueued,
           scheduled:  sidekiq_stats.scheduled_size,
           retries:    sidekiq_stats.retry_size,
+          dead:       sidekiq_stats.dead_size,
           default_latency: queue.latency,
         },
         redis: redis_stats

data/test/test_cli.rb

@@ -241,6 +241,35 @@ class TestCli < Sidekiq::Test
       end
     end
 
+    describe 'with an empty config file' do
+      before do
+        @tmp_file = Tempfile.new('sidekiq-test')
+        @tmp_path = @tmp_file.path
+        @tmp_file.close!
+      end
+
+      after do
+        File.unlink @tmp_path if File.exist? @tmp_path
+      end
+
+      it 'takes a path' do
+        @cli.parse(['sidekiq', '-C', @tmp_path])
+        assert_equal @tmp_path, Sidekiq.options[:config_file]
+      end
+
+      it 'should have an identical options hash, except for config_file' do
+        @cli.parse(['sidekiq'])
+        old_options = Sidekiq.options.clone
+
+        @cli.parse(['sidekiq', '-C', @tmp_path])
+        new_options = Sidekiq.options.clone
+        refute_equal old_options, new_options
+
+        new_options.delete(:config_file)
+        assert_equal old_options, new_options
+      end
+    end
+
     describe 'with config file and flags' do
       before do
         # We need an actual file here.

data/test/test_client.rb

@@ -105,13 +105,6 @@ class TestClient < Sidekiq::Test
       @redis.verify
     end
 
-    it 'handles perform_async on failure' do
-      @redis.expect :lpush, nil, ['queue:default', Array]
-      pushed = MyWorker.perform_async(1, 2)
-      refute pushed
-      @redis.verify
-    end
-
     it 'enqueues messages to redis' do
       @redis.expect :lpush, 1, ['queue:default', Array]
       pushed = Sidekiq::Client.enqueue(MyWorker, 1, 2)
@@ -266,6 +259,7 @@ class TestClient < Sidekiq::Test
     end
     it 'allows Resque helpers to point to different Redi' do
       conn = MiniTest::Mock.new
+      conn.expect(:multi, []) { |*args, &block| block.call }
       conn.expect(:zadd, 1, [String, Array])
       DWorker.sidekiq_options('pool' => ConnectionPool.new(size: 1) { conn })
       Sidekiq::Client.enqueue_in(10, DWorker, 3)

data/test/test_manager.rb

@@ -139,7 +139,7 @@ class TestManager < Sidekiq::Test
           info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
           assert_equal ["0"], info
           expires = Sidekiq.redis { |c| c.pttl('identity') }
-          assert_in_delta 60000, expires, 5
+          assert_in_delta 60000, expires, 50
         end
       end
     end

data/test/test_retry.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'helper'
 require 'sidekiq/scheduled'
 require 'sidekiq/middleware/server/retry_jobs'
@@ -50,6 +51,23 @@ class TestRetry < Sidekiq::Test
       @redis.verify
     end
 
+    it 'handles zany characters in error message, #1705' do
+      skip 'skipped! test requires ruby 2.1+' if RUBY_VERSION <= '2.1.0'
+      @redis.expect :zadd, 1, ['retry', String, String]
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => 2 }
+      msg2 = msg.dup
+      handler = Sidekiq::Middleware::Server::RetryJobs.new
+      assert_raises RuntimeError do
+        handler.call(worker, msg2, 'default') do
+          raise "kerblammo! #{195.chr}"
+        end
+      end
+      msg2.delete('failed_at')
+      assert_equal({"class"=>"Bob", "args"=>[1, 2, "foo"], "retry"=>2, "queue"=>"default", "error_message"=>"kerblammo! �", "error_class"=>"RuntimeError", "retry_count"=>0}, msg2)
+      @redis.verify
+    end
+
+
     it 'allows a max_retries option in initializer' do
       max_retries = 7
       1.upto(max_retries) do

data/test/test_web.rb

@@ -35,7 +35,7 @@ class TestWeb < Sidekiq::Test
         Sidekiq.redis do |conn|
           conn.incr('busy')
           conn.sadd('processes', 'foo:1234')
-          conn.hmset('foo:1234', 'info', Sidekiq.dump_json('hostname' => 'foo', 'started_at' => Time.now.to_f), 'at', Time.now.to_f, 'busy', 4)
+          conn.hmset('foo:1234', 'info', Sidekiq.dump_json('hostname' => 'foo', 'started_at' => Time.now.to_f, "queues" => []), 'at', Time.now.to_f, 'busy', 4)
           identity = 'foo:1234:workers'
           hash = {:queue => 'critical', :payload => { 'class' => WebWorker.name, 'args' => [1,'abc'] }, :run_at => Time.now.to_i }
           conn.hmset(identity, 1001, Sidekiq.dump_json(hash))
@@ -293,7 +293,7 @@ class TestWeb < Sidekiq::Test
       Sidekiq.redis do |conn|
         pro = 'foo:1234'
         conn.sadd('processes', pro)
-        conn.hmset(pro, 'info', Sidekiq.dump_json('started_at' => Time.now.to_f, 'labels' => ['frumduz']), 'busy', 1, 'beat', Time.now.to_f)
+        conn.hmset(pro, 'info', Sidekiq.dump_json('started_at' => Time.now.to_f, 'labels' => ['frumduz'], 'queues' =>[]), 'busy', 1, 'beat', Time.now.to_f)
         identity = "#{pro}:workers"
         hash = {:queue => 'critical', :payload => { 'class' => "FailWorker", 'args' => ["<a>hello</a>"] }, :run_at => Time.now.to_i }
         conn.hmset(identity, 100001, Sidekiq.dump_json(hash))

data/web/assets/javascripts/dashboard.js

@@ -214,6 +214,8 @@ var updateStatsSummary = function(data) {
   $('ul.summary li.scheduled span.count').html(data.scheduled.numberWithDelimiter())
   $('ul.summary li.retries span.count').html(data.retries.numberWithDelimiter())
   $('ul.summary li.enqueued span.count').html(data.enqueued.numberWithDelimiter())
+  $('ul.summary li.dead span.count').html(data.dead.numberWithDelimiter())
+
 }
 
 var updateRedisStats = function(data) {

data/web/assets/stylesheets/application.css

@@ -675,10 +675,12 @@ div.interval-slider input {
 .args {
   overflow-y: auto;
   max-height: 100px;
+  word-break: break-all;
 }
 .args-extended {
   overflow-y: scroll;
   max-height: 500px;
+  word-break: break-all;
 }
 
 

data/web/views/busy.erb

@@ -23,12 +23,15 @@
   </thead>
   <% Sidekiq::ProcessSet.new.each do |process| %>
     <tr>
-      <td>
+      <td width="50%">
         <%= "#{process['hostname']}:#{process['pid']}" %>
         <span class="label label-success"><%= process.tag %></span>
         <% process.labels.each do |label| %>
           <span class="label label-info"><%= label %></span>
         <% end %>
+        <br>
+        <b><%= "#{t('Queues')}: " %></b>
+        <%= process['queues'] * ", " %>
       </td>
       <td><%= relative_time(Time.at(process['started_at'])) %></td>
       <td><%= process['concurrency'] %></td>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.2.3
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-11 00:00:00.000000000 Z
+date: 2014-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -172,6 +172,10 @@ files:
 - Rakefile
 - bin/sidekiq
 - bin/sidekiqctl
+- lib/generators/sidekiq/templates/worker.rb.erb
+- lib/generators/sidekiq/templates/worker_spec.rb.erb
+- lib/generators/sidekiq/templates/worker_test.rb.erb
+- lib/generators/sidekiq/worker_generator.rb
 - lib/sidekiq.rb
 - lib/sidekiq/actor.rb
 - lib/sidekiq/api.rb
@@ -340,7 +344,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: Simple, efficient background processing for Ruby