sidekiq 0.6.0 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (45) hide show
  1. data/.travis.yml +3 -0
  2. data/Changes.md +15 -0
  3. data/README.md +1 -1
  4. data/TODO.md +0 -2
  5. data/examples/chef/cookbooks/sidekiq/README.rdoc +7 -0
  6. data/examples/chef/cookbooks/sidekiq/recipes/default.rb +54 -0
  7. data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb +8 -0
  8. data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb +219 -0
  9. data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb +28 -0
  10. data/examples/monitrc.conf +6 -0
  11. data/examples/por.rb +10 -0
  12. data/lib/sidekiq.rb +70 -0
  13. data/lib/sidekiq/cli.rb +72 -30
  14. data/lib/sidekiq/client.rb +16 -16
  15. data/lib/sidekiq/extensions/action_mailer.rb +27 -0
  16. data/lib/sidekiq/extensions/active_record.rb +29 -0
  17. data/lib/sidekiq/extensions/generic_proxy.rb +21 -0
  18. data/lib/sidekiq/manager.rb +16 -12
  19. data/lib/sidekiq/middleware/chain.rb +23 -23
  20. data/lib/sidekiq/middleware/client/resque_web_compatibility.rb +1 -4
  21. data/lib/sidekiq/middleware/client/unique_jobs.rb +5 -13
  22. data/lib/sidekiq/middleware/server/failure_jobs.rb +24 -0
  23. data/lib/sidekiq/middleware/server/unique_jobs.rb +1 -5
  24. data/lib/sidekiq/processor.rb +11 -13
  25. data/lib/sidekiq/redis_connection.rb +1 -1
  26. data/lib/sidekiq/util.rb +20 -1
  27. data/lib/sidekiq/version.rb +1 -1
  28. data/lib/sidekiq/worker.rb +1 -1
  29. data/myapp/Gemfile +0 -2
  30. data/myapp/app/controllers/work_controller.rb +25 -1
  31. data/myapp/app/mailers/user_mailer.rb +9 -0
  32. data/myapp/app/models/post.rb +3 -0
  33. data/myapp/app/views/user_mailer/greetings.html.erb +3 -0
  34. data/myapp/app/workers/hard_worker.rb +2 -2
  35. data/myapp/config/initializers/sidekiq.rb +6 -1
  36. data/myapp/config/routes.rb +3 -0
  37. data/sidekiq.gemspec +2 -0
  38. data/test/config.yml +11 -0
  39. data/test/test_cli.rb +119 -3
  40. data/test/test_client.rb +21 -10
  41. data/test/test_extensions.rb +45 -0
  42. data/test/test_manager.rb +2 -2
  43. data/test/test_middleware.rb +7 -12
  44. data/test/test_stats.rb +1 -2
  45. metadata +55 -16
data/.travis.yml ADDED
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
data/Changes.md CHANGED
@@ -1,3 +1,18 @@
1
+ 0.7.0
2
+ -----------
3
+
4
+ - Example chef recipe and monitrc script (jc00ke)
5
+ - Refactor global configuration into Sidekiq.configure\_server and
6
+ Sidekiq.configure\_client blocks. (mperham)
7
+ - Add optional middleware FailureJobs which saves failed jobs to a
8
+ 'failed' queue (fbjork)
9
+ - Upon shutdown, workers are now terminated after 5 seconds. This is to
10
+ meet Heroku's hard limit of 10 seconds for a process to shutdown. (mperham)
11
+ - Refactor middleware API for simplicity, see sidekiq/middleware/chain. (mperham)
12
+ - Add `delay` extensions for ActionMailer and ActiveRecord. (mperham)
13
+ - Added config file support. See test/config.yml for an example file. (jc00ke)
14
+ - Added pidfile for tools like monit (jc00ke)
15
+
1
16
  0.6.0
2
17
  -----------
3
18
 
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- Sidekiq
1
+ Sidekiq [![Build Status](https://secure.travis-ci.org/mperham/sidekiq.png)](http://travis-ci.org/mperham/sidekiq)
2
2
  ==============
3
3
 
4
4
  Simple, efficient message processing for Ruby.
data/TODO.md CHANGED
@@ -1,3 +1 @@
1
1
  - resque-ui-esque web ui
2
- - graceful shutdown (ideally Celluloid will provide this)
3
- - monit/god/etc example scripts
@@ -0,0 +1,7 @@
1
+ = DESCRIPTION:
2
+
3
+ Sidekiq is a Redis-backed Ruby library for creating background jobs, placing those jobs on multiple queues, and processing them later.
4
+
5
+ = USAGE:
6
+
7
+ add require_recipe "sidekiq" to main/recipes/default.rb
@@ -0,0 +1,54 @@
1
+ #
2
+ # Cookbook Name:: sidekiq
3
+ # Recipe:: default
4
+ #
5
+ if ['solo', 'util'].include?(node[:instance_role])
6
+
7
+ # for now
8
+ worker_count = 1
9
+
10
+ node[:applications].each do |app, data|
11
+ template "/etc/monit.d/sidekiq_#{app}.monitrc" do
12
+ owner 'root'
13
+ group 'root'
14
+ mode 0644
15
+ source "monitrc.conf.erb"
16
+ variables({
17
+ :num_workers => worker_count,
18
+ :app_name => app,
19
+ :rails_env => node[:environment][:framework_env]
20
+ })
21
+ end
22
+
23
+ template "/engineyard/bin/sidekiq" do
24
+ owner 'root'
25
+ group 'root'
26
+ mode 0755
27
+ source "sidekiq.erb"
28
+ end
29
+
30
+ worker_count.times do |count|
31
+ template "/data/#{app}/shared/config/sidekiq_#{count}.yml" do
32
+ owner node[:owner_name]
33
+ group node[:owner_name]
34
+ mode 0644
35
+ source "sidekiq.yml.erb"
36
+ variables({
37
+ :require => "/data/#{app}/current"
38
+ })
39
+ end
40
+ end
41
+
42
+ execute "ensure-sidekiq-is-setup-with-monit" do
43
+ command %Q{
44
+ monit reload
45
+ }
46
+ end
47
+
48
+ execute "restart-sidekiq" do
49
+ command %Q{
50
+ echo "sleep 20 && monit -g #{app}_sidekiq restart all" | at now
51
+ }
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,8 @@
1
+ <% (@num_workers || 1).times do |num| %>
2
+ check process sidekiq_<%= @app_name %>_<%= num %>
3
+ with pidfile /var/run/engineyard/sidekiq/<%= @app_name %>/sidekiq_<%= num %>.pid
4
+ start program = "/engineyard/bin/sidekiq <%= @app_name %> start <%= @rails_env %> sidekiq_<%= num %>.yml" with timeout 90 seconds
5
+ stop program = "/engineyard/bin/sidekiq <%= @app_name %> stop <%= @rails_env %> sidekiq_<%= num %>.yml" with timeout 90 seconds
6
+ if totalmem is greater than 300 MB for 2 cycles then restart # eating up memory?
7
+ group <%= @app_name %>_sidekiq
8
+ <% end %>
@@ -0,0 +1,219 @@
1
+ #!/bin/sh
2
+ #
3
+ # This script starts and stops the Sidekiq daemon
4
+ # This script belongs in /engineyard/bin/sidekiq
5
+ #
6
+
7
+ PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
8
+ CURDIR=`pwd`
9
+
10
+ usage() {
11
+ echo "Usage: $0 <appname> {start|stop|quit} <environment> <conf_file>"
12
+ echo -e "\nstop) is a synonym for quit"
13
+ echo "quit) issues -INT to request the worker to stop"
14
+ echo -e "\nSee http://mperham.github.com/sidekiq/ for more details"
15
+ exit 1
16
+ }
17
+
18
+ rm_lockfile(){
19
+ if [ -e $LOCK_FILE ]; then
20
+ logger -t "monit-sidekiq[$$]" "removing $LOCK_FILE for `cat $LOCK_FILE`"
21
+ rm $LOCK_FILE
22
+ fi
23
+ }
24
+
25
+ exit_cleanly() {
26
+ cd $CURDIR
27
+ logger -t "monit-sidekiq[$$]" "exiting wrapper cleanly with $RESULT"
28
+ exit $RESULT
29
+ }
30
+
31
+ unlock_and_exit_cleanly(){
32
+ rm_lockfile
33
+ exit_cleanly
34
+ }
35
+
36
+ set_pid_from_file(){
37
+ export PID=`cat $PID_FILE`
38
+ }
39
+
40
+ signal_worker() {
41
+ RESULT=0
42
+ if [ -f $PID_FILE ]; then
43
+ set_pid_from_file
44
+ logger -t "monit-sidekiq[$$]" "Issuing kill with -$SIG $PID"
45
+ SLEEP_COUNT=0
46
+ kill -$SIG $PID
47
+ fi
48
+ }
49
+
50
+ signal_worker_fatally(){
51
+ signal_worker()
52
+ if [ -f $PID_FILE ]; then
53
+ if [ -n "$ALLOW_TIMEOUT" ]; then
54
+ while [ -e /proc/$PID ]; do
55
+ sleep .25
56
+ let "SLEEP_COUNT+=1"
57
+ let "REPORT_TIME = $SLEEP_COUNT%4"
58
+ if(( "$SLEEP_COUNT" > $GRACE_TIME )); then
59
+ logger -t "monit-sidekiq[$$]" "Sidekiq worker with pid $PID for $WORKER_REF still running, issuing -TERM"
60
+ kill -15 $PID 2>/dev/null; true
61
+ elif(( $REPORT_TIME == 0 )); then
62
+ let "RUNTIME = $SLEEP_COUNT/4"
63
+ logger -t "monit-sidekiq[$$]" "waiting for $PID to die ( for $RUNTIME seconds now)"
64
+ fi
65
+ done
66
+ fi
67
+ sleep 1
68
+ if [ -d /proc/$PID ];then
69
+ for child in $(ps axo pid,ppid | awk "{ if ( \$2 == $PID ) { print \$1 }}");
70
+ do
71
+ kill -9 $child 2>/dev/null; true
72
+ logger -t "monit-sidekiq[$$]" "Murdering Sidekiq workers child with $child for $WORKER_REF"
73
+ break
74
+ done
75
+ while [ -d /proc/$PID ]; do
76
+ logger -t "monit-sidekiq[$$]" "Murdering Sidekiq worker with $PID for $WORKER_REF"
77
+ kill -9 $PID
78
+ sleep 1
79
+ done
80
+ fi
81
+ logger -t "monit-sidekiq[$$]" "Removing pid file for $PID - $WORKER_REF"
82
+ [ -e "$PID_FILE" -a ! -d /proc/$PID ] && rm -f $PID_FILE
83
+ fi
84
+ }
85
+
86
+ lock(){
87
+ RESULT=0
88
+ if [ -e $LOCK_FILE ]; then
89
+ LAST_LOCK_PID=`cat $LOCK_FILE`
90
+ if [ -n $LAST_LOCK_PID -a -z "`ps axo pid|grep $LAST_LOCK_PID`" -a -f $LOCK_FILE ];then
91
+ sleep 1
92
+ logger -t "monit-sidekiq[$$]" "Removing stale lock file for $WORKER_REF ($LAST_LOCK_PID)"
93
+ rm $LOCK_FILE 2>&1
94
+ else
95
+ logger -t "monit-sidekiq[$$]" "Monit already messing with $WORKER_REF ($LAST_LOCK_PID)"
96
+ RESULT=1
97
+ exit_cleanly
98
+ fi
99
+ fi
100
+ echo $$ > $LOCK_FILE
101
+ }
102
+
103
+ legacy_fix() {
104
+ #In the transition from 0.18.2 to 0.18.3 of ey monit scripts the way
105
+ #the pid file is used to find the process to kill has changed.
106
+ #To avert problems being left behind after an upgrade of this package,
107
+ if [ -f $PID_FILE ]; then
108
+ set_pid_from_file
109
+ if [ -n "`ps axo pid,command|grep $PID|grep 'su -c'`" ];then
110
+ logger -t "monit-sidekiq[$$]" "Monit Scripts have just been upgraded, killing old style workers"
111
+ for child in $(ps axo pid,ppid| awk "{ if ( \$2 == $PID ) { print \$1 }}");
112
+ do
113
+ kill -TERM $child 2> /dev/null
114
+ while [ -e /proc/$child ]; do
115
+ logger -t "monit-sidekiq[$$]" "killing legacy worker: $child"
116
+ [ -e /proc/$child ] && kill -9 $child 2> /dev/null
117
+ sleep 1
118
+ done
119
+ done
120
+ [ -e /proc/$PID ] && kill -9 $PID 2> /dev/null
121
+ rm $PID_FILE
122
+ unlock_exit_cleanly
123
+ fi
124
+ fi
125
+ }
126
+
127
+ if [ $# -lt 4 ]; then usage; fi
128
+
129
+ if [ "`whoami`" != "root" ]; then
130
+ logger -t `basename $0` -s "Must be run as root"
131
+ exit 1
132
+ fi
133
+
134
+ #Baisc Setup of default values
135
+ APP=$1 ; ACTION=$2; RACK_ENV=$3; CONF_FILE=$4;
136
+
137
+ APP_DIR="/data/${APP}"
138
+ APP_ROOT="${APP_DIR}/current"
139
+ APP_SHARED="${APP_DIR}/shared"
140
+ APP_CONFIG="${APP_SHARED}/config"
141
+
142
+ if [ -e "${APP_CONFIG}/${CONF_FILE}" ]; then
143
+ logger -t "sidekiq_${APP}" -s "Good, found a config file. Proceeding..."
144
+ else
145
+ logger -t "sidekiq_${APP}" -s "/data/${APP}/shared/config/${CONF_FILE} not found for app: ${APP}"
146
+ exit 1
147
+ fi
148
+
149
+ WORKER_REF=`echo $CONF_FILE | sed s/.yml.conf//`
150
+ LOG_FILE="$APP_ROOT/log/$WORKER_REF.log"
151
+ LOCK_FILE="/tmp/$WORKER_REF.monit-lock"
152
+ PID_FILE="/var/run/engineyard/sidekiq/$APP/$WORKER_REF.pid"
153
+ GEMFILE="$APP_ROOT/Gemfile"
154
+ SIDEKIQ="sidekiq"
155
+ if [ -f $GEMFILE ];then
156
+ SIDEKIQ="bundle exec $APP_ROOT/ey_bundler_binstubs/sidekiq"
157
+ fi
158
+
159
+ if [ -d $APP_ROOT ]; then
160
+ USER=$(stat -L -c"%U" $APP_ROOT)
161
+ export HOME="/home/$USER"
162
+
163
+ # Fix for SD-3786 - stop sending in VERBOSE= and VVERBOSE= by default
164
+ if declare -p VERBOSE >/dev/null 2>&1; then export V="VERBOSE=$VERBOSE"; fi
165
+ if declare -p VVERBOSE >/dev/null 2>&1; then export VV="VVERBOSE=$VVERBOSE"; fi
166
+
167
+ # Older versions of sudo need us to call env for the env vars to be set correctly
168
+ COMMAND="/usr/bin/env $V $VV APP_ROOT=${APP_ROOT} RACK_ENV=${RACK_ENV} RAILS_ENV=${RACK_ENV} $SIDEKIQ -e ${RACK_ENV} -C ${APP_CONFIG}/${CONF_FILE}"
169
+
170
+ if [ ! -d /var/run/engineyard/sidekiq/$APP ]; then
171
+ mkdir -p /var/run/engineyard/sidekiq/$APP
172
+ fi
173
+
174
+ # handle the second param, don't start if already existing
175
+
176
+ logger -t "monit-sidekiq[$$]" "${ACTION}ing Sidekiq worker $WORKER_REF"
177
+ case "$ACTION" in
178
+ start)
179
+ lock
180
+ cd $APP_ROOT
181
+ if [ -f $PID_FILE ]; then
182
+ set_pid_from_file
183
+ if [ -d /proc/$PID ]; then
184
+ logger -t "monit-sidekiq[$$]" "Sidekiq worker $WORKER_REF is already running with $PID."
185
+ RESULT=1
186
+ else
187
+ rm -f $PID_FILE
188
+ logger -t "monit-sidekiq[$$]" "Removing stale pid file ($PID_FILE) for pid $PID"
189
+ fi
190
+ fi
191
+ if [ ! -f $PID_FILE ]; then
192
+ sudo -u $USER -H $COMMAND >> $LOG_FILE 2>&1 &
193
+ RESULT=$?
194
+ logger -t "monit-sidekiq[$$]" "Started with pid $! and exit $RESULT"
195
+ echo $! > $PID_FILE
196
+ sleep .1
197
+ fi
198
+ unlock_and_exit_cleanly
199
+ ;;
200
+ stop|quit)
201
+ legacy_fix
202
+ lock
203
+ SIG="INT"
204
+ [ -z "$GRACE_TIME" ] && GRACE_TIME=60
205
+ ALLOW_TIMEOUT=1
206
+ signal_worker
207
+ [ -e "$LOCK_FILE" ] && rm $LOCK_FILE
208
+ unlock_and_exit_cleanly
209
+ ;;
210
+ *)
211
+ usage
212
+ exit_cleanly
213
+ ;;
214
+ esac
215
+ else
216
+ echo "/data/$APP/current doesn't exist."
217
+ usage
218
+ fi
219
+
@@ -0,0 +1,28 @@
1
+ ---
2
+ <% if @verbose %>
3
+ :verbose: <%= @verbose %>
4
+ <% end %>
5
+ <% if @namespace %>
6
+ :namespace: <%= @namespace %>
7
+ <% end %>
8
+ <% if @server %>
9
+ :server: <%= @server %>
10
+ <% end %>
11
+ <% if @environment %>
12
+ :environment: <%= @environment %>
13
+ <% end %>
14
+ <% if @require %>
15
+ :require: <%= @require %>
16
+ <% end %>
17
+ <% if @pidfile %>
18
+ :pidfile: <%= @pidfile %>
19
+ <% end %>
20
+ <% if @concurrency %>
21
+ :concurrency: <%= @concurrency %>
22
+ <% end %>
23
+ <% if @queues %>
24
+ :queues:
25
+ <% @queues.each do |name, priority| %>
26
+ - [<%= name %>, <%= priority %>]
27
+ <% end %>
28
+ <% end %>
@@ -0,0 +1,6 @@
1
+ check process sidekiq_myapp
2
+ with pidfile /path/to/sidekiq.pid
3
+ start program = "bundle exec sidekiq -C /path/to/sidekiq_conf.yml -P /path/to/sidekiq.pid" with timeout 90 seconds
4
+ stop program = "kill -s INT `cat /path/to/sidekiq.pid`" with timeout 90 seconds
5
+ if totalmem is greater than 200 MB for 2 cycles then restart # eating up memory?
6
+ group myapp_sidekiq
data/examples/por.rb CHANGED
@@ -1,5 +1,15 @@
1
1
  require 'sidekiq'
2
2
 
3
+ # If your client is single-threaded, we just need a single connection in our Redis connection pool
4
+ Sidekiq.configure_client do |config|
5
+ config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :size => 1, :url => 'redis://redis.host:1234/db')
6
+ end
7
+
8
+ # Sidekiq server is multi-threaded so our Redis connection pool size defaults to concurrency (-c)
9
+ Sidekiq.configure_server do |config|
10
+ config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :url => 'redis://redis.host:1234/db')
11
+ end
12
+
3
13
  # Start up sidekiq via
4
14
  # ./bin/sidekiq -r ./examples/por.rb
5
15
  # and then you can open up an IRB session like so:
data/lib/sidekiq.rb CHANGED
@@ -2,3 +2,73 @@ require 'sidekiq/version'
2
2
  require 'sidekiq/client'
3
3
  require 'sidekiq/worker'
4
4
  require 'sidekiq/rails' if defined?(::Rails)
5
+ require 'sidekiq/redis_connection'
6
+
7
+ require 'sidekiq/extensions/action_mailer' if defined?(::ActionMailer)
8
+ require 'sidekiq/extensions/active_record' if defined?(::ActiveRecord)
9
+
10
+ module Sidekiq
11
+
12
+ DEFAULTS = {
13
+ :queues => [],
14
+ :concurrency => 25,
15
+ :require => '.',
16
+ :environment => nil,
17
+ }
18
+
19
+ def self.options
20
+ @options ||= DEFAULTS.dup
21
+ end
22
+
23
+ def self.options=(opts)
24
+ @options = opts
25
+ end
26
+
27
+ ##
28
+ # Configuration for Sidekiq server, use like:
29
+ #
30
+ # Sidekiq.configure_server do |config|
31
+ # config.redis = Sidekiq::RedisConnection.create(:namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/mydb')
32
+ # config.server_middleware do |chain|
33
+ # chain.add MyServerHook
34
+ # end
35
+ # end
36
+ def self.configure_server
37
+ yield self if server?
38
+ end
39
+
40
+ ##
41
+ # Configuration for Sidekiq client, use like:
42
+ #
43
+ # Sidekiq.configure_client do |config|
44
+ # config.redis = Sidekiq::RedisConnection.create(:namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/mydb')
45
+ # end
46
+ def self.configure_client
47
+ yield self unless server?
48
+ end
49
+
50
+ def self.server?
51
+ defined?(Sidekiq::CLI)
52
+ end
53
+
54
+ def self.redis
55
+ @redis ||= Sidekiq::RedisConnection.create
56
+ end
57
+
58
+ def self.redis=(r)
59
+ @redis = r
60
+ end
61
+
62
+ def self.client_middleware
63
+ @client_chain ||= Client.default_middleware
64
+ yield @client_chain if block_given?
65
+ @client_chain
66
+ end
67
+
68
+ def self.server_middleware
69
+ @server_chain ||= Processor.default_middleware
70
+ yield @server_chain if block_given?
71
+ @server_chain
72
+ end
73
+
74
+ end