sidekiq 1.2.1 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

data/Changes.md CHANGED
@@ -1,3 +1,38 @@
1
+ 2.0.0
2
+ -----------
3
+
4
+ - **SCHEDULED JOBS**!
5
+
6
+ You can now use `perform_at` and `perform_in` to schedule jobs
7
+ to run at arbitrary points in the future, like so:
8
+
9
+ ```ruby
10
+ SomeWorker.perform_in(5.days, 'bob', 13)
11
+ SomeWorker.perform_at(5.days.from_now, 'bob', 13)
12
+ ```
13
+
14
+ It also works with the delay extensions:
15
+
16
+ ```ruby
17
+ UserMailer.delay_for(5.days).send_welcome_email(user.id)
18
+ ```
19
+
20
+ The time is approximately when the job will be placed on the queue;
21
+ it is not guaranteed to run at precisely at that moment in time.
22
+
23
+ This functionality is meant for one-off, arbitrary jobs. I still
24
+ recommend `whenever` or `clockwork` if you want cron-like,
25
+ recurring jobs. See `examples/scheduling.rb`
26
+
27
+ I want to specially thank @yabawock for his work on sidekiq-scheduler.
28
+ His extension for Sidekiq 1.x filled an obvious functional gap that I now think is
29
+ useful enough to implement in Sidekiq proper.
30
+
31
+ - Fixed issues due to Redis 3.x API changes. Sidekiq now requires
32
+ the Redis 3.x client.
33
+ - Inline testing now round trips arguments through JSON to catch
34
+ serialization issues (betelgeuse)
35
+
1
36
  1.2.1
2
37
  -----------
3
38
 
@@ -1,4 +1,4 @@
1
- # Sidekiq defers scheduling to other, better suited gems.
1
+ # Sidekiq defers scheduling cron-like tasks to other, better suited gems.
2
2
  # If you want to run a job regularly, here's an example
3
3
  # of using the 'whenever' gem to push jobs to Sidekiq
4
4
  # regularly.
@@ -9,6 +9,8 @@ require 'sidekiq/extensions/action_mailer'
9
9
  require 'sidekiq/extensions/active_record'
10
10
  require 'sidekiq/rails' if defined?(::Rails)
11
11
 
12
+ require 'multi_json'
13
+
12
14
  module Sidekiq
13
15
 
14
16
  DEFAULTS = {
@@ -10,23 +10,23 @@ Capistrano::Configuration.instance.load do
10
10
  namespace :sidekiq do
11
11
 
12
12
  desc "Quiet sidekiq (stop accepting new work)"
13
- task :quiet, :roles => lambda { fetch(:sidekiq_role) } do
13
+ task :quiet, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
14
14
  run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl quiet #{current_path}/tmp/pids/sidekiq.pid ; fi"
15
15
  end
16
16
 
17
17
  desc "Stop sidekiq"
18
- task :stop, :roles => lambda { fetch(:sidekiq_role) } do
18
+ task :stop, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
19
19
  run "cd #{current_path} && if [ -f #{current_path}/tmp/pids/sidekiq.pid ]; then #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl stop #{current_path}/tmp/pids/sidekiq.pid #{fetch :sidekiq_timeout} ; fi"
20
20
  end
21
21
 
22
22
  desc "Start sidekiq"
23
- task :start, :roles => lambda { fetch(:sidekiq_role) } do
23
+ task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
24
24
  rails_env = fetch(:rails_env, "production")
25
25
  run "cd #{current_path} ; nohup #{fetch(:bundle_cmd, "bundle")} exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{current_path}/tmp/pids/sidekiq.pid >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
26
26
  end
27
27
 
28
28
  desc "Restart sidekiq"
29
- task :restart, :roles => lambda { fetch(:sidekiq_role) } do
29
+ task :restart, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
30
30
  stop
31
31
  start
32
32
  end
@@ -1,12 +1,12 @@
1
1
  trap 'INT' do
2
2
  # Handle Ctrl-C in JRuby like MRI
3
3
  # http://jira.codehaus.org/browse/JRUBY-4637
4
- Thread.main.raise Interrupt
4
+ Sidekiq::CLI.instance.interrupt
5
5
  end
6
6
 
7
7
  trap 'TERM' do
8
8
  # Heroku sends TERM and then waits 10 seconds for process to exit.
9
- Thread.main.raise Interrupt
9
+ Sidekiq::CLI.instance.interrupt
10
10
  end
11
11
 
12
12
  trap 'USR1' do
@@ -30,7 +30,7 @@ require 'celluloid'
30
30
  require 'sidekiq'
31
31
  require 'sidekiq/util'
32
32
  require 'sidekiq/manager'
33
- require 'sidekiq/retry'
33
+ require 'sidekiq/scheduled'
34
34
 
35
35
  module Sidekiq
36
36
  class CLI
@@ -43,6 +43,8 @@ module Sidekiq
43
43
 
44
44
  def initialize
45
45
  @code = nil
46
+ @interrupt_mutex = Mutex.new
47
+ @interrupted = false
46
48
  end
47
49
 
48
50
  def parse(args=ARGV)
@@ -63,7 +65,7 @@ module Sidekiq
63
65
 
64
66
  def run
65
67
  @manager = Sidekiq::Manager.new(options)
66
- poller = Sidekiq::Retry::Poller.new
68
+ poller = Sidekiq::Scheduled::Poller.new
67
69
  begin
68
70
  logger.info 'Starting processing, hit Ctrl-C to stop'
69
71
  @manager.start!
@@ -80,6 +82,15 @@ module Sidekiq
80
82
  end
81
83
  end
82
84
 
85
+ def interrupt
86
+ @interrupt_mutex.synchronize do
87
+ unless @interrupted
88
+ @interrupted = true
89
+ Thread.main.raise Interrupt
90
+ end
91
+ end
92
+ end
93
+
83
94
  private
84
95
 
85
96
  def die(code)
@@ -1,5 +1,3 @@
1
- require 'multi_json'
2
-
3
1
  require 'sidekiq/middleware/chain'
4
2
  require 'sidekiq/middleware/client/unique_jobs'
5
3
 
@@ -50,9 +48,13 @@ module Sidekiq
50
48
  Sidekiq.client_middleware.invoke(worker_class, item, queue) do
51
49
  payload = Sidekiq.dump_json(item)
52
50
  Sidekiq.redis do |conn|
53
- _, pushed = conn.multi do
54
- conn.sadd('queues', queue)
55
- conn.rpush("queue:#{queue}", payload)
51
+ if item['at']
52
+ pushed = (conn.zadd('schedule', item['at'].to_s, payload) == 1)
53
+ else
54
+ _, pushed = conn.multi do
55
+ conn.sadd('queues', queue)
56
+ conn.rpush("queue:#{queue}", payload)
57
+ end
56
58
  end
57
59
  end
58
60
  end
@@ -3,12 +3,16 @@ require 'sidekiq/extensions/generic_proxy'
3
3
  module Sidekiq
4
4
  module Extensions
5
5
  ##
6
- # Adds a 'delay' method to ActionMailer to offload arbitrary email
6
+ # Adds 'delay' and 'delay_for' to ActionMailer to offload arbitrary email
7
7
  # delivery to Sidekiq. Example:
8
8
  #
9
9
  # UserMailer.delay.send_welcome_email(new_user)
10
+ # UserMailer.delay_for(5.days).send_welcome_email(new_user)
10
11
  class DelayedMailer
11
12
  include Sidekiq::Worker
13
+ # I think it's reasonable to assume that emails should take less
14
+ # than 30 seconds to send.
15
+ sidekiq_options :timeout => 30
12
16
 
13
17
  def perform(yml)
14
18
  (target, method_name, args) = YAML.load(yml)
@@ -20,6 +24,9 @@ module Sidekiq
20
24
  def delay
21
25
  Proxy.new(DelayedMailer, self)
22
26
  end
27
+ def delay_for(interval)
28
+ Proxy.new(DelayedMailer, self, Time.now.to_f + interval.to_f)
29
+ end
23
30
  end
24
31
 
25
32
  end
@@ -21,6 +21,9 @@ module Sidekiq
21
21
  def delay
22
22
  Proxy.new(DelayedModel, self)
23
23
  end
24
+ def delay_for(interval)
25
+ Proxy.new(DelayedModel, self, Time.now.to_f + interval.to_f)
26
+ end
24
27
  end
25
28
 
26
29
  end
@@ -1,9 +1,10 @@
1
1
  module Sidekiq
2
2
  module Extensions
3
3
  class Proxy < (RUBY_VERSION < '1.9' ? Object : BasicObject)
4
- def initialize(performable, target)
4
+ def initialize(performable, target, at=nil)
5
5
  @performable = performable
6
6
  @target = target
7
+ @at = at
7
8
  end
8
9
 
9
10
  def method_missing(name, *args)
@@ -13,7 +14,11 @@ module Sidekiq
13
14
  # to JSON and then deserialized on the other side back into a
14
15
  # Ruby object.
15
16
  obj = [@target, name, args]
16
- @performable.perform_async(::YAML.dump(obj))
17
+ if @at
18
+ @performable.perform_at(@at, ::YAML.dump(obj))
19
+ else
20
+ @performable.perform_async(::YAML.dump(obj))
21
+ end
17
22
  end
18
23
  end
19
24
 
@@ -1,5 +1,4 @@
1
1
  require 'celluloid'
2
- require 'multi_json'
3
2
 
4
3
  require 'sidekiq/util'
5
4
  require 'sidekiq/processor'
@@ -1,5 +1,4 @@
1
1
  require 'digest'
2
- require 'multi_json'
3
2
 
4
3
  module Sidekiq
5
4
  module Middleware
@@ -1,5 +1,3 @@
1
- require 'multi_json'
2
-
3
1
  module Sidekiq
4
2
  module Middleware
5
3
  module Server
@@ -1,12 +1,21 @@
1
- require 'multi_json'
2
-
3
- require 'sidekiq/retry'
1
+ require 'sidekiq/scheduled'
4
2
 
5
3
  module Sidekiq
6
4
  module Middleware
7
5
  module Server
8
6
  ##
9
7
  # Automatically retry jobs that fail in Sidekiq.
8
+ # Sidekiq's retry support assumes a typical development lifecycle:
9
+ # 0. push some code changes with a bug in it
10
+ # 1. bug causes message processing to fail, sidekiq's middleware captures
11
+ # the message and pushes it onto a retry queue
12
+ # 2. sidekiq retries messages in the retry queue multiple times with
13
+ # an exponential delay, the message continues to fail
14
+ # 3. after a few days, a developer deploys a fix. the message is
15
+ # reprocessed successfully.
16
+ # 4. if 3 never happens, sidekiq will eventually give up and throw the
17
+ # message away.
18
+ #
10
19
  # A message looks like:
11
20
  #
12
21
  # { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'] }
@@ -24,7 +33,10 @@ module Sidekiq
24
33
  # to the message and everyone is using Airbrake, right?
25
34
  class RetryJobs
26
35
  include Sidekiq::Util
27
- include Sidekiq::Retry
36
+
37
+ # delayed_job uses the same basic formula
38
+ MAX_COUNT = 25
39
+ DELAY = proc { |count| (count ** 4) + 15 }
28
40
 
29
41
  def call(worker, msg, queue)
30
42
  yield
@@ -1,5 +1,3 @@
1
- require 'multi_json'
2
-
3
1
  module Sidekiq
4
2
  module Middleware
5
3
  module Server
@@ -1,5 +1,4 @@
1
1
  require 'celluloid'
2
- require 'multi_json'
3
2
  require 'sidekiq/util'
4
3
 
5
4
  require 'sidekiq/middleware/server/active_record'
@@ -0,0 +1,47 @@
1
+ require 'sidekiq'
2
+ require 'sidekiq/util'
3
+ require 'celluloid'
4
+
5
+ module Sidekiq
6
+ module Scheduled
7
+
8
+ POLL_INTERVAL = 15
9
+
10
+ ##
11
+ # The Poller checks Redis every N seconds for messages in the retry or scheduled
12
+ # set have passed their timestamp and should be enqueued. If so, it
13
+ # just pops the message back onto its original queue so the
14
+ # workers can pick it up like any other message.
15
+ class Poller
16
+ include Celluloid
17
+ include Sidekiq::Util
18
+
19
+ SETS = %w(retry schedule)
20
+
21
+ def poll
22
+ watchdog('scheduling poller thread died!') do
23
+ # A message's "score" in Redis is the time at which it should be processed.
24
+ # Just check Redis for the set of messages with a timestamp before now.
25
+ now = Time.now.to_f.to_s
26
+ Sidekiq.redis do |conn|
27
+ SETS.each do |sorted_set|
28
+ (messages, _) = conn.multi do
29
+ conn.zrangebyscore(sorted_set, '-inf', now)
30
+ conn.zremrangebyscore(sorted_set, '-inf', now)
31
+ end
32
+
33
+ messages.each do |message|
34
+ logger.debug { "enqueued #{sorted_set}: #{message}" }
35
+ msg = Sidekiq.load_json(message)
36
+ conn.rpush("queue:#{msg['queue']}", message)
37
+ end
38
+ end
39
+ end
40
+
41
+ after(POLL_INTERVAL) { poll }
42
+ end
43
+ end
44
+
45
+ end
46
+ end
47
+ end
@@ -29,7 +29,7 @@ module Sidekiq
29
29
  module ClassMethods
30
30
  alias_method :perform_async_old, :perform_async
31
31
  def perform_async(*args)
32
- new.perform(*args)
32
+ new.perform(*Sidekiq.load_json(Sidekiq.dump_json(args)))
33
33
  true
34
34
  end
35
35
  end
@@ -1,3 +1,3 @@
1
1
  module Sidekiq
2
- VERSION = "1.2.1"
2
+ VERSION = "2.0.0"
3
3
  end
@@ -1,8 +1,6 @@
1
1
  require 'sinatra/base'
2
2
  require 'slim'
3
3
  require 'sprockets'
4
- require 'multi_json'
5
-
6
4
  module Sidekiq
7
5
  class SprocketsMiddleware
8
6
  def initialize(app, options={})
@@ -57,14 +55,22 @@ module Sidekiq
57
55
  Sidekiq.redis { |conn| conn.get('stat:failed') } || 0
58
56
  end
59
57
 
60
- def retry_count
61
- Sidekiq.redis { |conn| conn.zcard('retry') }
58
+ def zcard(name)
59
+ Sidekiq.redis { |conn| conn.zcard(name) }
62
60
  end
63
61
 
64
62
  def retries(count=50)
63
+ zcontents('retry', count)
64
+ end
65
+
66
+ def scheduled(count=50)
67
+ zcontents('schedule', count)
68
+ end
69
+
70
+ def zcontents(name, count)
65
71
  Sidekiq.redis do |conn|
66
- results = conn.zrange('retry', 0, count, :withscores => true)
67
- results.each_slice(2).map { |msg, score| [Sidekiq.load_json(msg), Float(score)] }
72
+ results = conn.zrange(name, 0, count, :withscores => true)
73
+ results.map { |msg, score| [Sidekiq.load_json(msg), score] }
68
74
  end
69
75
  end
70
76
 
@@ -76,6 +82,10 @@ module Sidekiq
76
82
  end
77
83
  end
78
84
 
85
+ def backlog
86
+ queues.map {|name, size| size }.inject(0) {|memo, val| memo + val }
87
+ end
88
+
79
89
  def retries_with_score(score)
80
90
  Sidekiq.redis do |conn|
81
91
  results = conn.zrangebyscore('retry', score, score)
@@ -138,14 +148,29 @@ module Sidekiq
138
148
  slim :retries
139
149
  end
140
150
 
151
+ get '/scheduled' do
152
+ @scheduled = scheduled
153
+ slim :scheduled
154
+ end
155
+
156
+ post '/scheduled' do
157
+ halt 404 unless params[:score]
158
+ halt 404 unless params['delete']
159
+ params[:score].each do |score|
160
+ s = score.to_f
161
+ process_score('schedule', s, :delete)
162
+ end
163
+ redirect root_path
164
+ end
165
+
141
166
  post '/retries' do
142
167
  halt 404 unless params[:score]
143
168
  params[:score].each do |score|
144
169
  s = score.to_f
145
170
  if params['retry']
146
- process_score(s, :retry)
171
+ process_score('retry', s, :retry)
147
172
  elsif params['delete']
148
- process_score(s, :delete)
173
+ process_score('retry', s, :delete)
149
174
  end
150
175
  end
151
176
  redirect root_path
@@ -162,20 +187,21 @@ module Sidekiq
162
187
  redirect root_path
163
188
  end
164
189
 
165
- def process_score(score, operation)
190
+ def process_score(set, score, operation)
166
191
  case operation
167
192
  when :retry
168
193
  Sidekiq.redis do |conn|
169
- results = conn.zrangebyscore('retry', score, score)
170
- conn.zremrangebyscore('retry', score, score)
194
+ results = conn.zrangebyscore(set, score, score)
195
+ conn.zremrangebyscore(set, score, score)
171
196
  results.map do |message|
172
197
  msg = Sidekiq.load_json(message)
173
- conn.rpush("queue:#{msg['queue']}", message)
198
+ msg['retry_count'] = msg['retry_count'] - 1
199
+ conn.rpush("queue:#{msg['queue']}", Sidekiq.dump_json(msg))
174
200
  end
175
201
  end
176
202
  when :delete
177
203
  Sidekiq.redis do |conn|
178
- conn.zremrangebyscore('retry', score, score)
204
+ conn.zremrangebyscore(set, score, score)
179
205
  end
180
206
  end
181
207
  end
@@ -33,6 +33,13 @@ module Sidekiq
33
33
  Sidekiq::Client.push('class' => self, 'args' => args)
34
34
  end
35
35
 
36
+ def perform_in(interval, *args)
37
+ int = interval.to_f
38
+ ts = (int < 1_000_000_000 ? Time.now.to_f + int : int)
39
+ Sidekiq::Client.push('class' => self, 'args' => args, 'at' => ts)
40
+ end
41
+ alias_method :perform_at, :perform_in
42
+
36
43
  ##
37
44
  # Allows customization for this type of Worker.
38
45
  # Legal options:
@@ -8,7 +8,7 @@ class WorkController < ApplicationController
8
8
  end
9
9
 
10
10
  def email
11
- UserMailer.delay.greetings(Time.now)
11
+ UserMailer.delay_for(30.seconds).greetings(Time.now)
12
12
  render :nothing => true
13
13
  end
14
14
 
@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
13
13
  gem.name = "sidekiq"
14
14
  gem.require_paths = ["lib"]
15
15
  gem.version = Sidekiq::VERSION
16
- gem.add_dependency 'redis'
16
+ gem.add_dependency 'redis', '~> 3'
17
17
  gem.add_dependency 'redis-namespace'
18
18
  gem.add_dependency 'connection_pool', '~> 0.9.0'
19
- gem.add_dependency 'celluloid', '~> 0.10.0'
19
+ gem.add_dependency 'celluloid', '~> 0.11.0'
20
20
  gem.add_dependency 'multi_json', '~> 1'
21
- gem.add_development_dependency 'minitest'
21
+ gem.add_development_dependency 'minitest', '~> 3'
22
22
  gem.add_development_dependency 'sinatra'
23
23
  gem.add_development_dependency 'slim'
24
24
  gem.add_development_dependency 'rake'
@@ -29,6 +29,12 @@ class TestExtensions < MiniTest::Unit::TestCase
29
29
  assert_equal 1, Sidekiq.redis {|c| c.llen('queue:default') }
30
30
  end
31
31
 
32
+ it 'allows delayed scheduling of AR class methods' do
33
+ assert_equal 0, Sidekiq.redis {|c| c.zcard('schedule') }
34
+ MyModel.delay_for(5.days).long_class_method
35
+ assert_equal 1, Sidekiq.redis {|c| c.zcard('schedule') }
36
+ end
37
+
32
38
  class UserMailer < ActionMailer::Base
33
39
  def greetings(a, b)
34
40
  raise "Should not be called!"
@@ -42,6 +48,12 @@ class TestExtensions < MiniTest::Unit::TestCase
42
48
  assert_equal ['default'], Sidekiq::Client.registered_queues
43
49
  assert_equal 1, Sidekiq.redis {|c| c.llen('queue:default') }
44
50
  end
51
+
52
+ it 'allows delayed scheduling of AM mails' do
53
+ assert_equal 0, Sidekiq.redis {|c| c.zcard('schedule') }
54
+ UserMailer.delay_for(5.days).greetings(1, 2)
55
+ assert_equal 1, Sidekiq.redis {|c| c.zcard('schedule') }
56
+ end
45
57
  end
46
58
 
47
59
  describe 'sidekiq rails extensions configuration' do
@@ -1,6 +1,5 @@
1
1
  require 'helper'
2
- require 'multi_json'
3
- require 'sidekiq/retry'
2
+ require 'sidekiq/scheduled'
4
3
  require 'sidekiq/middleware/server/retry_jobs'
5
4
 
6
5
  class TestRetry < MiniTest::Unit::TestCase
@@ -113,9 +112,10 @@ class TestRetry < MiniTest::Unit::TestCase
113
112
  it 'should poll like a bad mother...SHUT YO MOUTH' do
114
113
  fake_msg = Sidekiq.dump_json({ 'class' => 'Bob', 'args' => [1,2], 'queue' => 'someq' })
115
114
  @redis.expect :multi, [[fake_msg], 1], []
115
+ @redis.expect :multi, [[], nil], []
116
116
  @redis.expect :rpush, 1, ['queue:someq', fake_msg]
117
117
 
118
- inst = Sidekiq::Retry::Poller.new
118
+ inst = Sidekiq::Scheduled::Poller.new
119
119
  inst.poll
120
120
 
121
121
  @redis.verify
@@ -0,0 +1,33 @@
1
+ require 'helper'
2
+ require 'sidekiq/scheduled'
3
+
4
+ class TestScheduling < MiniTest::Unit::TestCase
5
+ describe 'middleware' do
6
+ before do
7
+ @redis = MiniTest::Mock.new
8
+ # Ugh, this is terrible.
9
+ Sidekiq.instance_variable_set(:@redis, @redis)
10
+
11
+ def @redis.with; yield self; end
12
+ end
13
+
14
+ class ScheduledWorker
15
+ include Sidekiq::Worker
16
+ def perform(x)
17
+ end
18
+ end
19
+
20
+ it 'schedules a job via interval' do
21
+ @redis.expect :zadd, 1, ['schedule', String, String]
22
+ assert_equal true, ScheduledWorker.perform_in(600, 'mike')
23
+ @redis.verify
24
+ end
25
+
26
+ it 'schedules a job via timestamp' do
27
+ @redis.expect :zadd, 1, ['schedule', String, String]
28
+ assert_equal true, ScheduledWorker.perform_in(5.days.from_now, 'mike')
29
+ @redis.verify
30
+ end
31
+ end
32
+
33
+ end
@@ -12,6 +12,7 @@ Sidekiq.hook_rails!
12
12
  class TestInline < MiniTest::Unit::TestCase
13
13
  describe 'sidekiq inline testing' do
14
14
  class InlineError < RuntimeError; end
15
+ class ParameterIsNotString < RuntimeError; end
15
16
 
16
17
  class InlineWorker
17
18
  include Sidekiq::Worker
@@ -20,6 +21,13 @@ class TestInline < MiniTest::Unit::TestCase
20
21
  end
21
22
  end
22
23
 
24
+ class InlineWorkerWithTimeParam
25
+ include Sidekiq::Worker
26
+ def perform(time)
27
+ raise ParameterIsNotString unless time.is_a?(String)
28
+ end
29
+ end
30
+
23
31
  class InlineFooMailer < ActionMailer::Base
24
32
  def bar(str)
25
33
  raise InlineError
@@ -71,5 +79,9 @@ class TestInline < MiniTest::Unit::TestCase
71
79
  Sidekiq::Client.enqueue(InlineWorker, false)
72
80
  end
73
81
  end
82
+
83
+ it 'should relay parameters through json' do
84
+ assert Sidekiq::Client.enqueue(InlineWorkerWithTimeParam, Time.now)
85
+ end
74
86
  end
75
87
  end
@@ -80,6 +80,20 @@ class TestWeb < MiniTest::Unit::TestCase
80
80
  end
81
81
  end
82
82
 
83
+ it 'can display scheduled' do
84
+ get '/scheduled'
85
+ assert_equal 200, last_response.status
86
+ assert_match /found/, last_response.body
87
+ refute_match /HardWorker/, last_response.body
88
+
89
+ add_scheduled
90
+
91
+ get '/scheduled'
92
+ assert_equal 200, last_response.status
93
+ refute_match /found/, last_response.body
94
+ assert_match /HardWorker/, last_response.body
95
+ end
96
+
83
97
  it 'can display retries' do
84
98
  get '/retries'
85
99
  assert_equal 200, last_response.status
@@ -104,6 +118,17 @@ class TestWeb < MiniTest::Unit::TestCase
104
118
  assert_match /HardWorker/, last_response.body
105
119
  end
106
120
 
121
+ def add_scheduled
122
+ msg = { 'class' => 'HardWorker',
123
+ 'args' => ['bob', 1, Time.now.to_f],
124
+ 'at' => Time.now.to_f }
125
+ score = Time.now.to_f
126
+ Sidekiq.redis do |conn|
127
+ conn.zadd('schedule', score, Sidekiq.dump_json(msg))
128
+ end
129
+ [msg, score]
130
+ end
131
+
107
132
  def add_retry
108
133
  msg = { 'class' => 'HardWorker',
109
134
  'args' => ['bob', 1, Time.now.to_f],
@@ -4,8 +4,9 @@
4
4
  p Processed: #{processed}
5
5
  p Failed: #{failed}
6
6
  p Busy Workers: #{workers.size}
7
- p Retries Pending: #{retry_count}
8
- p Queue Backlog: #{queues.map{|q,size| size}.sum}
7
+ p Scheduled: #{zcard('schedule')}
8
+ p Retries Pending: #{zcard('retry')}
9
+ p Queue Backlog: #{backlog}
9
10
 
10
11
  .tabbable
11
12
  ul.nav.nav-tabs
@@ -14,6 +14,8 @@ html
14
14
  a href='#{{root_path}}' Home
15
15
  li
16
16
  a href='#{{root_path}}retries' Retries
17
+ li
18
+ a href='#{{root_path}}scheduled' Scheduled
17
19
  ul.nav.pull-right
18
20
  li
19
21
  a Redis: #{location}
@@ -0,0 +1,25 @@
1
+ h1 Scheduled Jobs
2
+
3
+ - if @scheduled.size > 0
4
+ form action="#{root_path}scheduled" method="post"
5
+ table class="table table-striped table-bordered"
6
+ tr
7
+ th
8
+ input type="checkbox" class="check_all"
9
+ th When
10
+ th Queue
11
+ th Worker
12
+ th Args
13
+ - @scheduled.each do |(msg, score)|
14
+ tr
15
+ td
16
+ input type='checkbox' name='score[]' value='#{score}'
17
+ td== relative_time(Time.at(score))
18
+ td
19
+ a href="#{root_path}queues/#{msg['queue']}" #{msg['queue']}
20
+ td= msg['class']
21
+ td= display_args(msg['args'])
22
+ input.btn.btn-danger type="submit" name="delete" value="Delete"
23
+ - else
24
+ p No scheduled jobs found.
25
+ a href="#{root_path}" Back
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 2.0.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,22 +9,22 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-05-16 00:00:00.000000000 Z
12
+ date: 2012-06-01 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: redis
16
- requirement: &70113151847120 !ruby/object:Gem::Requirement
16
+ requirement: &70308439006360 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
- - - ! '>='
19
+ - - ~>
20
20
  - !ruby/object:Gem::Version
21
- version: '0'
21
+ version: '3'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70113151847120
24
+ version_requirements: *70308439006360
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: redis-namespace
27
- requirement: &70113151845720 !ruby/object:Gem::Requirement
27
+ requirement: &70308439005860 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70113151845720
35
+ version_requirements: *70308439005860
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: connection_pool
38
- requirement: &70113151844280 !ruby/object:Gem::Requirement
38
+ requirement: &70308439005040 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ~>
@@ -43,21 +43,21 @@ dependencies:
43
43
  version: 0.9.0
44
44
  type: :runtime
45
45
  prerelease: false
46
- version_requirements: *70113151844280
46
+ version_requirements: *70308439005040
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: celluloid
49
- requirement: &70113151842720 !ruby/object:Gem::Requirement
49
+ requirement: &70308438999880 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ~>
53
53
  - !ruby/object:Gem::Version
54
- version: 0.10.0
54
+ version: 0.11.0
55
55
  type: :runtime
56
56
  prerelease: false
57
- version_requirements: *70113151842720
57
+ version_requirements: *70308438999880
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: multi_json
60
- requirement: &70113151841480 !ruby/object:Gem::Requirement
60
+ requirement: &70308438998500 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ~>
@@ -65,21 +65,21 @@ dependencies:
65
65
  version: '1'
66
66
  type: :runtime
67
67
  prerelease: false
68
- version_requirements: *70113151841480
68
+ version_requirements: *70308438998500
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: minitest
71
- requirement: &70113151840480 !ruby/object:Gem::Requirement
71
+ requirement: &70308438997400 !ruby/object:Gem::Requirement
72
72
  none: false
73
73
  requirements:
74
- - - ! '>='
74
+ - - ~>
75
75
  - !ruby/object:Gem::Version
76
- version: '0'
76
+ version: '3'
77
77
  type: :development
78
78
  prerelease: false
79
- version_requirements: *70113151840480
79
+ version_requirements: *70308438997400
80
80
  - !ruby/object:Gem::Dependency
81
81
  name: sinatra
82
- requirement: &70113151839320 !ruby/object:Gem::Requirement
82
+ requirement: &70308438996900 !ruby/object:Gem::Requirement
83
83
  none: false
84
84
  requirements:
85
85
  - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
87
87
  version: '0'
88
88
  type: :development
89
89
  prerelease: false
90
- version_requirements: *70113151839320
90
+ version_requirements: *70308438996900
91
91
  - !ruby/object:Gem::Dependency
92
92
  name: slim
93
- requirement: &70113151838740 !ruby/object:Gem::Requirement
93
+ requirement: &70308438996320 !ruby/object:Gem::Requirement
94
94
  none: false
95
95
  requirements:
96
96
  - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
98
98
  version: '0'
99
99
  type: :development
100
100
  prerelease: false
101
- version_requirements: *70113151838740
101
+ version_requirements: *70308438996320
102
102
  - !ruby/object:Gem::Dependency
103
103
  name: rake
104
- requirement: &70113151837560 !ruby/object:Gem::Requirement
104
+ requirement: &70308438995720 !ruby/object:Gem::Requirement
105
105
  none: false
106
106
  requirements:
107
107
  - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
109
109
  version: '0'
110
110
  type: :development
111
111
  prerelease: false
112
- version_requirements: *70113151837560
112
+ version_requirements: *70308438995720
113
113
  - !ruby/object:Gem::Dependency
114
114
  name: actionmailer
115
- requirement: &70113151833660 !ruby/object:Gem::Requirement
115
+ requirement: &70308438994740 !ruby/object:Gem::Requirement
116
116
  none: false
117
117
  requirements:
118
118
  - - ~>
@@ -120,10 +120,10 @@ dependencies:
120
120
  version: '3'
121
121
  type: :development
122
122
  prerelease: false
123
- version_requirements: *70113151833660
123
+ version_requirements: *70308438994740
124
124
  - !ruby/object:Gem::Dependency
125
125
  name: activerecord
126
- requirement: &70113151829900 !ruby/object:Gem::Requirement
126
+ requirement: &70308438992940 !ruby/object:Gem::Requirement
127
127
  none: false
128
128
  requirements:
129
129
  - - ~>
@@ -131,7 +131,7 @@ dependencies:
131
131
  version: '3'
132
132
  type: :development
133
133
  prerelease: false
134
- version_requirements: *70113151829900
134
+ version_requirements: *70308438992940
135
135
  description: Simple, efficient message processing for Ruby
136
136
  email:
137
137
  - mperham@gmail.com
@@ -187,7 +187,7 @@ files:
187
187
  - lib/sidekiq/processor.rb
188
188
  - lib/sidekiq/rails.rb
189
189
  - lib/sidekiq/redis_connection.rb
190
- - lib/sidekiq/retry.rb
190
+ - lib/sidekiq/scheduled.rb
191
191
  - lib/sidekiq/testing.rb
192
192
  - lib/sidekiq/testing/inline.rb
193
193
  - lib/sidekiq/util.rb
@@ -245,6 +245,7 @@ files:
245
245
  - test/test_middleware.rb
246
246
  - test/test_processor.rb
247
247
  - test/test_retry.rb
248
+ - test/test_scheduling.rb
248
249
  - test/test_stats.rb
249
250
  - test/test_testing.rb
250
251
  - test/test_testing_inline.rb
@@ -275,6 +276,7 @@ files:
275
276
  - web/views/queue.slim
276
277
  - web/views/retries.slim
277
278
  - web/views/retry.slim
279
+ - web/views/scheduled.slim
278
280
  homepage: http://mperham.github.com/sidekiq
279
281
  licenses: []
280
282
  post_install_message:
@@ -310,6 +312,7 @@ test_files:
310
312
  - test/test_middleware.rb
311
313
  - test/test_processor.rb
312
314
  - test/test_retry.rb
315
+ - test/test_scheduling.rb
313
316
  - test/test_stats.rb
314
317
  - test/test_testing.rb
315
318
  - test/test_testing_inline.rb
@@ -1,59 +0,0 @@
1
- require 'sidekiq'
2
- require 'sidekiq/util'
3
- require 'celluloid'
4
- require 'multi_json'
5
-
6
- module Sidekiq
7
- ##
8
- # Sidekiq's retry support assumes a typical development lifecycle:
9
- # 0. push some code changes with a bug in it
10
- # 1. bug causes message processing to fail, sidekiq's middleware captures
11
- # the message and pushes it onto a retry queue
12
- # 2. sidekiq retries messages in the retry queue multiple times with
13
- # an exponential delay, the message continues to fail
14
- # 3. after a few days, a developer deploys a fix. the message is
15
- # reprocessed successfully.
16
- # 4. if 3 never happens, sidekiq will eventually give up and throw the
17
- # message away.
18
- module Retry
19
-
20
- # delayed_job uses the same basic formula
21
- MAX_COUNT = 25
22
- DELAY = proc { |count| (count ** 4) + 15 }
23
- POLL_INTERVAL = 15
24
-
25
- ##
26
- # The Poller checks Redis every N seconds for messages in the retry
27
- # set have passed their retry timestamp and should be retried. If so, it
28
- # just pops the message back onto its original queue so the
29
- # workers can pick it up like any other message.
30
- class Poller
31
- include Celluloid
32
- include Sidekiq::Util
33
-
34
- def poll
35
- watchdog('retry poller thread died!') do
36
-
37
- Sidekiq.redis do |conn|
38
- # A message's "score" in Redis is the time at which it should be retried.
39
- # Just check Redis for the set of messages with a timestamp before now.
40
- messages = nil
41
- now = Time.now.to_f.to_s
42
- (messages, _) = conn.multi do
43
- conn.zrangebyscore('retry', '-inf', now)
44
- conn.zremrangebyscore('retry', '-inf', now)
45
- end
46
-
47
- messages.each do |message|
48
- logger.debug { "Retrying #{message}" }
49
- msg = Sidekiq.load_json(message)
50
- conn.rpush("queue:#{msg['queue']}", message)
51
- end
52
- end
53
-
54
- after(POLL_INTERVAL) { poll }
55
- end
56
- end
57
- end
58
- end
59
- end