sidekiq 3.2.2 → 3.2.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ac4be785d484b8190a4aeb09fc93d21f4897f2f1
4
- data.tar.gz: 9955d71f3356939e2cc06f7287f66d850b18604c
3
+ metadata.gz: 56f90dbb4ffba7f12619e992cdd36e92c0c26c90
4
+ data.tar.gz: c5a9d6faf0ae1d1aa6b2ecb65b2adc282032a5aa
5
5
  SHA512:
6
- metadata.gz: ba6263b604d50445ad7582a49870daf4af2105570d0d43fe6702128903bdda651c640435eff1d4f61fdeb828193ed91fc6552b536f87fc8cee3d4b313b7c13f9
7
- data.tar.gz: 260002fc92ba5a50aa910402f388ca1f77f2c21aab076fcd59f8e9d04c68e48cbd2b2889f89ef1caee745e9a585af86f1ea9caa65c46af68983d23f66ec0e55c
6
+ metadata.gz: 730a5cd70e49741c0426ee7f7d467939f72adae0ea276c00cc45cf331e48794fe8c40d1d84c4432c0d92c20bf7f795c06be775c2cae26e98170967673922c302
7
+ data.tar.gz: 6261ef3999c2a4ad2674172ce1c84d4bef1fb172c0952994aa0232f943e7671add7c72984f381b3c4a7198a723f72cc3362abc6a81baee136e162992ee63f3b7
data/Changes.md CHANGED
@@ -1,6 +1,18 @@
1
+ 3.2.3
2
+ -----------
3
+
4
+ - Clean invalid bytes from error message before converting to JSON (requires Ruby 2.1+) [#1705]
5
+ - Add queues list for each process to the Busy page. [davetoxa, #1897]
6
+ - Fix for crash caused by empty config file. [jordan0day, #1901]
7
+ - Add Rails Worker generator, `rails g sidekiq:worker User` will create `app/workers/user_worker.rb`. [seuros, #1909]
8
+ - Fix Web UI rendering with huge job arguments [jhass, #1918]
9
+ - Minor refactoring of Sidekiq::Client internals, for Sidekiq Pro. [#1919]
10
+
1
11
  3.2.2
2
12
  -----------
3
13
 
14
+ - **This version of Sidekiq will no longer start on Ruby 1.9.** Sidekiq
15
+ 3 does not support MRI 1.9 but we've allowed it to run before now.
4
16
  - Fix issue which could cause Sidekiq workers to disappear from the Busy
5
17
  tab while still being active [#1884]
6
18
  - Add "Back to App" button in Web UI. You can set the button link via
@@ -3,10 +3,25 @@ Sidekiq Pro Changelog
3
3
 
4
4
  Please see [http://sidekiq.org/pro](http://sidekiq.org/pro) for more details and how to buy.
5
5
 
6
+ 1.8.0
7
+ -----------
8
+
9
+ - Fix long standing Batch race condition where Batches can complete
10
+ before they have been fully defined or only half-defined. Requires
11
+ Sidekiq 3.2.3. [#1919]
12
+
13
+
14
+ 1.7.6
15
+ -----------
16
+
17
+ - Quick release to verify #1919
18
+
19
+
6
20
  1.7.5
7
21
  -----------
8
22
 
9
23
  - Fix job filtering within the Dead tab.
24
+ - Add APIs and wiki documentation for invalidating jobs within a batch.
10
25
 
11
26
 
12
27
  1.7.4
@@ -0,0 +1,9 @@
1
+ <% module_namespacing do -%>
2
+ class <%= class_name %>Worker
3
+ include Sidekiq::Worker
4
+
5
+ def perform(*args)
6
+ # Do something
7
+ end
8
+ end
9
+ <% end -%>
@@ -0,0 +1,6 @@
1
+ require 'rails_helper'
2
+ <% module_namespacing do -%>
3
+ RSpec.describe <%= class_name %>Worker, :type => :worker do
4
+ pending "add some examples to (or delete) #{__FILE__}"
5
+ end
6
+ <% end -%>
@@ -0,0 +1,8 @@
1
+ require_relative 'test_helper'
2
+ <% module_namespacing do -%>
3
+ class <%= class_name %>WorkerTest < MiniTest::Unit::TestCase
4
+ def test_example
5
+ skip "add some examples to (or delete) #{__FILE__}"
6
+ end
7
+ end
8
+ <% end -%>
@@ -0,0 +1,49 @@
1
+ require 'rails/generators/named_base'
2
+
3
+ module Sidekiq
4
+ module Generators # :nodoc:
5
+ class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
7
+
8
+ check_class_collision suffix: 'Worker'
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_worker_file
15
+ template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ if defined?(RSpec)
20
+ create_worker_spec
21
+ else
22
+ create_worker_test
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ def create_worker_spec
29
+ template_file = File.join(
30
+ 'spec/workers',
31
+ class_path,
32
+ "#{file_name}_spec.rb"
33
+ )
34
+ template 'worker_spec.rb.erb', template_file
35
+ end
36
+
37
+ def create_worker_test
38
+ template_file = File.join(
39
+ 'test/workers',
40
+ class_path,
41
+ "#{file_name}_test.rb"
42
+ )
43
+ template 'worker_test.rb.erb', template_file
44
+ end
45
+
46
+
47
+ end
48
+ end
49
+ end
@@ -326,7 +326,7 @@ module Sidekiq
326
326
  Sidekiq.redis do |conn|
327
327
  conn.multi do
328
328
  conn.zadd('dead', now, message)
329
- conn.zremrangebyscore('dead', '-inf', now - DeadSet::DEAD_JOB_TIMEOUT)
329
+ conn.zremrangebyscore('dead', '-inf', now - DeadSet::TIMEOUT)
330
330
  conn.zremrangebyrank('dead', 0, - DeadSet::MAX_JOBS)
331
331
  end
332
332
  end
@@ -520,7 +520,7 @@ module Sidekiq
520
520
  # Allows enumeration of dead jobs within Sidekiq.
521
521
  #
522
522
  class DeadSet < JobSet
523
- DEAD_JOB_TIMEOUT = 180 * 24 * 60 * 60 # 6 months
523
+ TIMEOUT = 180 * 24 * 60 * 60 # 6 months
524
524
  MAX_JOBS = 10_000
525
525
 
526
526
  def initialize
@@ -353,7 +353,7 @@ module Sidekiq
353
353
  def parse_config(cfile)
354
354
  opts = {}
355
355
  if File.exist?(cfile)
356
- opts = YAML.load(ERB.new(IO.read(cfile)).result)
356
+ opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
357
357
  opts = opts.merge(opts.delete(environment) || {})
358
358
  parse_queues(opts, opts.delete(:queues) || [])
359
359
  else
@@ -176,23 +176,26 @@ module Sidekiq
176
176
  private
177
177
 
178
178
  def raw_push(payloads)
179
- pushed = false
180
179
  @redis_pool.with do |conn|
181
- if payloads.first['at']
182
- pushed = conn.zadd('schedule', payloads.map do |hash|
183
- at = hash.delete('at').to_s
184
- [at, Sidekiq.dump_json(hash)]
185
- end)
186
- else
187
- q = payloads.first['queue']
188
- to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
189
- _, pushed = conn.multi do
190
- conn.sadd('queues', q)
191
- conn.lpush("queue:#{q}", to_push)
192
- end
180
+ conn.multi do
181
+ atomic_push(conn, payloads)
193
182
  end
194
183
  end
195
- pushed
184
+ true
185
+ end
186
+
187
+ def atomic_push(conn, payloads)
188
+ if payloads.first['at']
189
+ conn.zadd('schedule', payloads.map do |hash|
190
+ at = hash.delete('at').to_s
191
+ [at, Sidekiq.dump_json(hash)]
192
+ end)
193
+ else
194
+ q = payloads.first['queue']
195
+ to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
196
+ conn.sadd('queues', q)
197
+ conn.lpush("queue:#{q}", to_push)
198
+ end
196
199
  end
197
200
 
198
201
  def process_single(worker_class, item)
@@ -49,7 +49,7 @@ module Sidekiq
49
49
  poller.terminate if poller.alive?
50
50
 
51
51
  manager.async.stop(:shutdown => true, :timeout => @options[:timeout])
52
- manager.wait(:shutdown)
52
+ manager.wait(:shutdown) if manager.alive?
53
53
 
54
54
  # Requeue everything in case there was a worker who grabbed work while stopped
55
55
  Sidekiq::Fetcher.strategy.bulk_requeue([], @options)
@@ -1,3 +1,9 @@
1
+ #
2
+ # Simple middleware to save the current locale and restore it when the job executes.
3
+ # Use it by requiring it in your initializer:
4
+ #
5
+ # require 'sidekiq/middleware/i18n'
6
+ #
1
7
  module Sidekiq::Middleware::I18n
2
8
  # Get the current locale and store it in the message
3
9
  # to be sent to Sidekiq.
@@ -1,4 +1,5 @@
1
1
  require 'sidekiq/scheduled'
2
+ require 'sidekiq/api'
2
3
 
3
4
  module Sidekiq
4
5
  module Middleware
@@ -86,7 +87,16 @@ module Sidekiq
86
87
  else
87
88
  queue
88
89
  end
89
- msg['error_message'] = e.message[0..10_000]
90
+
91
+ # App code can stuff all sorts of crazy binary data into the error message
92
+ # that won't convert to JSON.
93
+ m = e.message[0..10_000]
94
+ if m.respond_to?(:scrub!)
95
+ m.force_encoding("utf-8")
96
+ m.scrub!
97
+ end
98
+
99
+ msg['error_message'] = m
90
100
  msg['error_class'] = e.class.name
91
101
  count = if msg['retry_count']
92
102
  msg['retried_at'] = Time.now.to_f
@@ -122,9 +132,6 @@ module Sidekiq
122
132
 
123
133
  private
124
134
 
125
- DEAD_JOB_TIMEOUT = 180 * 24 * 60 * 60 # 6 months
126
- MAX_JOBS = 10_000
127
-
128
135
  def retries_exhausted(worker, msg)
129
136
  logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
130
137
  begin
@@ -145,7 +152,7 @@ module Sidekiq
145
152
  Sidekiq.redis do |conn|
146
153
  conn.multi do
147
154
  conn.zadd('dead', now, payload)
148
- conn.zremrangebyscore('dead', '-inf', now - DeadSet::DEAD_JOB_TIMEOUT)
155
+ conn.zremrangebyscore('dead', '-inf', now - DeadSet::TIMEOUT)
149
156
  conn.zremrangebyrank('dead', 0, -DeadSet::MAX_JOBS)
150
157
  end
151
158
  end
@@ -1,3 +1,3 @@
1
1
  module Sidekiq
2
- VERSION = "3.2.2"
2
+ VERSION = "3.2.3"
3
3
  end
@@ -221,6 +221,7 @@ module Sidekiq
221
221
  enqueued: sidekiq_stats.enqueued,
222
222
  scheduled: sidekiq_stats.scheduled_size,
223
223
  retries: sidekiq_stats.retry_size,
224
+ dead: sidekiq_stats.dead_size,
224
225
  default_latency: queue.latency,
225
226
  },
226
227
  redis: redis_stats
@@ -241,6 +241,35 @@ class TestCli < Sidekiq::Test
241
241
  end
242
242
  end
243
243
 
244
+ describe 'with an empty config file' do
245
+ before do
246
+ @tmp_file = Tempfile.new('sidekiq-test')
247
+ @tmp_path = @tmp_file.path
248
+ @tmp_file.close!
249
+ end
250
+
251
+ after do
252
+ File.unlink @tmp_path if File.exist? @tmp_path
253
+ end
254
+
255
+ it 'takes a path' do
256
+ @cli.parse(['sidekiq', '-C', @tmp_path])
257
+ assert_equal @tmp_path, Sidekiq.options[:config_file]
258
+ end
259
+
260
+ it 'should have an identical options hash, except for config_file' do
261
+ @cli.parse(['sidekiq'])
262
+ old_options = Sidekiq.options.clone
263
+
264
+ @cli.parse(['sidekiq', '-C', @tmp_path])
265
+ new_options = Sidekiq.options.clone
266
+ refute_equal old_options, new_options
267
+
268
+ new_options.delete(:config_file)
269
+ assert_equal old_options, new_options
270
+ end
271
+ end
272
+
244
273
  describe 'with config file and flags' do
245
274
  before do
246
275
  # We need an actual file here.
@@ -105,13 +105,6 @@ class TestClient < Sidekiq::Test
105
105
  @redis.verify
106
106
  end
107
107
 
108
- it 'handles perform_async on failure' do
109
- @redis.expect :lpush, nil, ['queue:default', Array]
110
- pushed = MyWorker.perform_async(1, 2)
111
- refute pushed
112
- @redis.verify
113
- end
114
-
115
108
  it 'enqueues messages to redis' do
116
109
  @redis.expect :lpush, 1, ['queue:default', Array]
117
110
  pushed = Sidekiq::Client.enqueue(MyWorker, 1, 2)
@@ -266,6 +259,7 @@ class TestClient < Sidekiq::Test
266
259
  end
267
260
  it 'allows Resque helpers to point to different Redi' do
268
261
  conn = MiniTest::Mock.new
262
+ conn.expect(:multi, []) { |*args, &block| block.call }
269
263
  conn.expect(:zadd, 1, [String, Array])
270
264
  DWorker.sidekiq_options('pool' => ConnectionPool.new(size: 1) { conn })
271
265
  Sidekiq::Client.enqueue_in(10, DWorker, 3)
@@ -139,7 +139,7 @@ class TestManager < Sidekiq::Test
139
139
  info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
140
140
  assert_equal ["0"], info
141
141
  expires = Sidekiq.redis { |c| c.pttl('identity') }
142
- assert_in_delta 60000, expires, 5
142
+ assert_in_delta 60000, expires, 50
143
143
  end
144
144
  end
145
145
  end
@@ -1,3 +1,4 @@
1
+ # encoding: utf-8
1
2
  require 'helper'
2
3
  require 'sidekiq/scheduled'
3
4
  require 'sidekiq/middleware/server/retry_jobs'
@@ -50,6 +51,23 @@ class TestRetry < Sidekiq::Test
50
51
  @redis.verify
51
52
  end
52
53
 
54
+ it 'handles zany characters in error message, #1705' do
55
+ skip 'skipped! test requires ruby 2.1+' if RUBY_VERSION <= '2.1.0'
56
+ @redis.expect :zadd, 1, ['retry', String, String]
57
+ msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => 2 }
58
+ msg2 = msg.dup
59
+ handler = Sidekiq::Middleware::Server::RetryJobs.new
60
+ assert_raises RuntimeError do
61
+ handler.call(worker, msg2, 'default') do
62
+ raise "kerblammo! #{195.chr}"
63
+ end
64
+ end
65
+ msg2.delete('failed_at')
66
+ assert_equal({"class"=>"Bob", "args"=>[1, 2, "foo"], "retry"=>2, "queue"=>"default", "error_message"=>"kerblammo! �", "error_class"=>"RuntimeError", "retry_count"=>0}, msg2)
67
+ @redis.verify
68
+ end
69
+
70
+
53
71
  it 'allows a max_retries option in initializer' do
54
72
  max_retries = 7
55
73
  1.upto(max_retries) do
@@ -35,7 +35,7 @@ class TestWeb < Sidekiq::Test
35
35
  Sidekiq.redis do |conn|
36
36
  conn.incr('busy')
37
37
  conn.sadd('processes', 'foo:1234')
38
- conn.hmset('foo:1234', 'info', Sidekiq.dump_json('hostname' => 'foo', 'started_at' => Time.now.to_f), 'at', Time.now.to_f, 'busy', 4)
38
+ conn.hmset('foo:1234', 'info', Sidekiq.dump_json('hostname' => 'foo', 'started_at' => Time.now.to_f, "queues" => []), 'at', Time.now.to_f, 'busy', 4)
39
39
  identity = 'foo:1234:workers'
40
40
  hash = {:queue => 'critical', :payload => { 'class' => WebWorker.name, 'args' => [1,'abc'] }, :run_at => Time.now.to_i }
41
41
  conn.hmset(identity, 1001, Sidekiq.dump_json(hash))
@@ -293,7 +293,7 @@ class TestWeb < Sidekiq::Test
293
293
  Sidekiq.redis do |conn|
294
294
  pro = 'foo:1234'
295
295
  conn.sadd('processes', pro)
296
- conn.hmset(pro, 'info', Sidekiq.dump_json('started_at' => Time.now.to_f, 'labels' => ['frumduz']), 'busy', 1, 'beat', Time.now.to_f)
296
+ conn.hmset(pro, 'info', Sidekiq.dump_json('started_at' => Time.now.to_f, 'labels' => ['frumduz'], 'queues' =>[]), 'busy', 1, 'beat', Time.now.to_f)
297
297
  identity = "#{pro}:workers"
298
298
  hash = {:queue => 'critical', :payload => { 'class' => "FailWorker", 'args' => ["<a>hello</a>"] }, :run_at => Time.now.to_i }
299
299
  conn.hmset(identity, 100001, Sidekiq.dump_json(hash))
@@ -214,6 +214,8 @@ var updateStatsSummary = function(data) {
214
214
  $('ul.summary li.scheduled span.count').html(data.scheduled.numberWithDelimiter())
215
215
  $('ul.summary li.retries span.count').html(data.retries.numberWithDelimiter())
216
216
  $('ul.summary li.enqueued span.count').html(data.enqueued.numberWithDelimiter())
217
+ $('ul.summary li.dead span.count').html(data.dead.numberWithDelimiter())
218
+
217
219
  }
218
220
 
219
221
  var updateRedisStats = function(data) {
@@ -675,10 +675,12 @@ div.interval-slider input {
675
675
  .args {
676
676
  overflow-y: auto;
677
677
  max-height: 100px;
678
+ word-break: break-all;
678
679
  }
679
680
  .args-extended {
680
681
  overflow-y: scroll;
681
682
  max-height: 500px;
683
+ word-break: break-all;
682
684
  }
683
685
 
684
686
 
@@ -23,12 +23,15 @@
23
23
  </thead>
24
24
  <% Sidekiq::ProcessSet.new.each do |process| %>
25
25
  <tr>
26
- <td>
26
+ <td width="50%">
27
27
  <%= "#{process['hostname']}:#{process['pid']}" %>
28
28
  <span class="label label-success"><%= process.tag %></span>
29
29
  <% process.labels.each do |label| %>
30
30
  <span class="label label-info"><%= label %></span>
31
31
  <% end %>
32
+ <br>
33
+ <b><%= "#{t('Queues')}: " %></b>
34
+ <%= process['queues'] * ", " %>
32
35
  </td>
33
36
  <td><%= relative_time(Time.at(process['started_at'])) %></td>
34
37
  <td><%= process['concurrency'] %></td>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.2
4
+ version: 3.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mike Perham
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-08-11 00:00:00.000000000 Z
11
+ date: 2014-09-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -172,6 +172,10 @@ files:
172
172
  - Rakefile
173
173
  - bin/sidekiq
174
174
  - bin/sidekiqctl
175
+ - lib/generators/sidekiq/templates/worker.rb.erb
176
+ - lib/generators/sidekiq/templates/worker_spec.rb.erb
177
+ - lib/generators/sidekiq/templates/worker_test.rb.erb
178
+ - lib/generators/sidekiq/worker_generator.rb
175
179
  - lib/sidekiq.rb
176
180
  - lib/sidekiq/actor.rb
177
181
  - lib/sidekiq/api.rb
@@ -340,7 +344,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
340
344
  version: '0'
341
345
  requirements: []
342
346
  rubyforge_project:
343
- rubygems_version: 2.2.2
347
+ rubygems_version: 2.4.1
344
348
  signing_key:
345
349
  specification_version: 4
346
350
  summary: Simple, efficient background processing for Ruby