sidekiq 0.6.0 → 0.7.0

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/Changes.md

@@ -1,3 +1,18 @@
+0.7.0
+-----------
+
+- Example chef recipe and monitrc script (jc00ke)
+- Refactor global configuration into Sidekiq.configure\_server and
+  Sidekiq.configure\_client blocks. (mperham)
+- Add optional middleware FailureJobs which saves failed jobs to a
+  'failed' queue (fbjork)
+- Upon shutdown, workers are now terminated after 5 seconds.  This is to
+  meet Heroku's hard limit of 10 seconds for a process to shutdown. (mperham)
+- Refactor middleware API for simplicity, see sidekiq/middleware/chain. (mperham)
+- Add `delay` extensions for ActionMailer and ActiveRecord. (mperham)
+- Added config file support. See test/config.yml for an example file.  (jc00ke)
+- Added pidfile for tools like monit (jc00ke)
+
 0.6.0
 -----------
 

data/README.md

@@ -1,4 +1,4 @@
-Sidekiq
+Sidekiq [![Build Status](https://secure.travis-ci.org/mperham/sidekiq.png)](http://travis-ci.org/mperham/sidekiq)
 ==============
 
 Simple, efficient message processing for Ruby.

data/TODO.md

@@ -1,3 +1 @@
 - resque-ui-esque web ui
-- graceful shutdown (ideally Celluloid will provide this)
-- monit/god/etc example scripts

data/examples/chef/cookbooks/sidekiq/README.rdoc

@@ -0,0 +1,7 @@
+= DESCRIPTION:
+
+Sidekiq is a Redis-backed Ruby library for creating background jobs, placing those jobs on multiple queues, and processing them later.
+
+= USAGE: 
+
+add require_recipe "sidekiq" to main/recipes/default.rb

data/examples/chef/cookbooks/sidekiq/recipes/default.rb

@@ -0,0 +1,54 @@
+#
+# Cookbook Name:: sidekiq
+# Recipe:: default
+#
+if ['solo', 'util'].include?(node[:instance_role])
+
+  # for now
+  worker_count = 1
+
+  node[:applications].each do |app, data|
+    template "/etc/monit.d/sidekiq_#{app}.monitrc" do 
+      owner 'root' 
+      group 'root' 
+      mode 0644 
+      source "monitrc.conf.erb" 
+      variables({ 
+        :num_workers => worker_count,
+        :app_name => app, 
+        :rails_env => node[:environment][:framework_env] 
+      }) 
+    end
+
+    template "/engineyard/bin/sidekiq" do
+      owner 'root'
+      group 'root' 
+      mode 0755
+      source "sidekiq.erb" 
+    end
+
+    worker_count.times do |count|
+      template "/data/#{app}/shared/config/sidekiq_#{count}.yml" do
+        owner node[:owner_name]
+        group node[:owner_name]
+        mode 0644
+        source "sidekiq.yml.erb"
+        variables({
+          :require => "/data/#{app}/current"
+        })
+      end
+    end
+
+    execute "ensure-sidekiq-is-setup-with-monit" do 
+      command %Q{ 
+        monit reload 
+      } 
+    end
+
+    execute "restart-sidekiq" do 
+      command %Q{ 
+        echo "sleep 20 && monit -g #{app}_sidekiq restart all" | at now 
+      }
+    end
+  end 
+end

data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb

@@ -0,0 +1,8 @@
+<% (@num_workers || 1).times do |num| %>
+check process sidekiq_<%= @app_name %>_<%= num %>
+  with pidfile /var/run/engineyard/sidekiq/<%= @app_name %>/sidekiq_<%= num %>.pid
+  start program = "/engineyard/bin/sidekiq <%= @app_name %> start <%= @rails_env  %> sidekiq_<%= num %>.yml" with timeout 90 seconds
+  stop program = "/engineyard/bin/sidekiq <%= @app_name %> stop <%= @rails_env  %> sidekiq_<%= num %>.yml" with timeout 90 seconds
+  if totalmem is greater than 300 MB for 2 cycles then restart # eating up memory?
+  group <%= @app_name %>_sidekiq
+<% end %>

data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb

@@ -0,0 +1,219 @@
+#!/bin/sh
+#
+# This script starts and stops the Sidekiq daemon
+# This script belongs in /engineyard/bin/sidekiq
+#
+
+PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
+CURDIR=`pwd`
+
+usage() {
+  echo "Usage: $0 <appname> {start|stop|quit} <environment> <conf_file>"
+  echo -e "\nstop)     is a synonym for quit"
+  echo "quit)     issues -INT to request the worker to stop"
+  echo -e "\nSee http://mperham.github.com/sidekiq/ for more details"
+  exit 1
+}
+
+rm_lockfile(){
+  if [ -e $LOCK_FILE ]; then
+   logger -t "monit-sidekiq[$$]" "removing $LOCK_FILE for `cat $LOCK_FILE`"
+   rm $LOCK_FILE
+  fi
+}
+
+exit_cleanly() {
+  cd $CURDIR
+  logger -t "monit-sidekiq[$$]" "exiting wrapper cleanly with $RESULT"
+  exit $RESULT
+}
+
+unlock_and_exit_cleanly(){
+  rm_lockfile
+  exit_cleanly
+}
+
+set_pid_from_file(){
+  export PID=`cat $PID_FILE`
+}
+
+signal_worker() {
+  RESULT=0
+  if [ -f $PID_FILE ]; then
+    set_pid_from_file
+    logger -t "monit-sidekiq[$$]" "Issuing kill with -$SIG $PID"
+    SLEEP_COUNT=0
+    kill -$SIG $PID 
+  fi
+}
+
+signal_worker_fatally(){
+  signal_worker() 
+  if [ -f $PID_FILE ]; then
+    if [ -n "$ALLOW_TIMEOUT" ]; then
+      while [ -e /proc/$PID ]; do
+        sleep .25
+        let "SLEEP_COUNT+=1"
+        let "REPORT_TIME = $SLEEP_COUNT%4"
+        if(( "$SLEEP_COUNT" > $GRACE_TIME )); then
+          logger -t "monit-sidekiq[$$]" "Sidekiq worker with pid $PID for $WORKER_REF still running, issuing -TERM"
+          kill -15 $PID 2>/dev/null; true
+        elif(( $REPORT_TIME == 0 )); then
+          let "RUNTIME = $SLEEP_COUNT/4"
+          logger -t "monit-sidekiq[$$]" "waiting for $PID to die ( for $RUNTIME seconds now)"
+        fi 
+      done
+    fi
+    sleep 1
+    if [ -d /proc/$PID ];then 
+      for child in $(ps axo pid,ppid | awk "{ if ( \$2 == $PID ) { print \$1 }}");
+      do
+        kill -9 $child 2>/dev/null; true
+        logger -t "monit-sidekiq[$$]" "Murdering Sidekiq workers child with $child for $WORKER_REF"
+        break
+      done    
+      while [ -d /proc/$PID ]; do
+        logger -t "monit-sidekiq[$$]" "Murdering Sidekiq worker with $PID for $WORKER_REF"
+        kill -9 $PID
+        sleep 1
+      done
+    fi
+    logger -t "monit-sidekiq[$$]" "Removing pid file for $PID - $WORKER_REF"
+    [ -e "$PID_FILE" -a ! -d /proc/$PID ] && rm -f $PID_FILE
+  fi
+}
+
+lock(){
+  RESULT=0
+  if [ -e $LOCK_FILE ]; then
+    LAST_LOCK_PID=`cat $LOCK_FILE`
+    if [ -n $LAST_LOCK_PID -a -z "`ps axo pid|grep $LAST_LOCK_PID`" -a -f $LOCK_FILE ];then
+      sleep 1
+      logger -t "monit-sidekiq[$$]" "Removing stale lock file for $WORKER_REF ($LAST_LOCK_PID)"
+      rm $LOCK_FILE 2>&1
+    else
+      logger -t "monit-sidekiq[$$]" "Monit already messing with $WORKER_REF ($LAST_LOCK_PID)"
+      RESULT=1
+      exit_cleanly 
+    fi
+  fi
+  echo $$ > $LOCK_FILE
+}
+
+legacy_fix() {
+ #In the transition from 0.18.2 to 0.18.3 of ey monit scripts the way 
+ #the pid file is used to find the process to kill has changed. 
+ #To avert problems being left behind after an upgrade of this package,
+  if [ -f $PID_FILE ]; then
+    set_pid_from_file
+    if [ -n "`ps axo pid,command|grep $PID|grep 'su -c'`" ];then
+      logger -t "monit-sidekiq[$$]" "Monit Scripts have just been upgraded, killing old style workers"
+      for child in $(ps axo pid,ppid| awk "{ if ( \$2 == $PID ) { print \$1 }}");
+      do
+        kill -TERM $child 2> /dev/null
+        while [ -e /proc/$child ]; do
+          logger -t "monit-sidekiq[$$]" "killing legacy worker: $child"
+          [ -e /proc/$child ] && kill -9 $child 2> /dev/null
+	  sleep 1 
+        done
+      done
+      [ -e /proc/$PID   ] && kill -9 $PID   2> /dev/null
+      rm $PID_FILE
+      unlock_exit_cleanly  
+    fi
+  fi
+}
+
+if [ $# -lt 4 ]; then usage; fi
+
+if [ "`whoami`" != "root" ]; then
+  logger -t `basename $0` -s "Must be run as root" 
+  exit 1
+fi
+
+#Baisc Setup of default values
+APP=$1 ; ACTION=$2; RACK_ENV=$3; CONF_FILE=$4;
+
+APP_DIR="/data/${APP}"
+APP_ROOT="${APP_DIR}/current"
+APP_SHARED="${APP_DIR}/shared"
+APP_CONFIG="${APP_SHARED}/config"
+
+if [ -e "${APP_CONFIG}/${CONF_FILE}" ]; then
+  logger -t "sidekiq_${APP}" -s "Good, found a config file. Proceeding..."
+else
+  logger -t "sidekiq_${APP}" -s "/data/${APP}/shared/config/${CONF_FILE} not found for app: ${APP}"
+  exit 1
+fi
+
+WORKER_REF=`echo $CONF_FILE | sed s/.yml.conf//`
+LOG_FILE="$APP_ROOT/log/$WORKER_REF.log"
+LOCK_FILE="/tmp/$WORKER_REF.monit-lock"
+PID_FILE="/var/run/engineyard/sidekiq/$APP/$WORKER_REF.pid"
+GEMFILE="$APP_ROOT/Gemfile"
+SIDEKIQ="sidekiq"
+if [ -f $GEMFILE ];then
+  SIDEKIQ="bundle exec $APP_ROOT/ey_bundler_binstubs/sidekiq"
+fi
+
+if [ -d $APP_ROOT ]; then
+  USER=$(stat -L -c"%U" $APP_ROOT)
+  export HOME="/home/$USER" 
+
+  # Fix for SD-3786 - stop sending in VERBOSE= and VVERBOSE= by default
+  if declare -p VERBOSE >/dev/null 2>&1; then export V="VERBOSE=$VERBOSE"; fi
+  if declare -p VVERBOSE >/dev/null 2>&1; then export VV="VVERBOSE=$VVERBOSE"; fi
+
+  # Older versions of sudo need us to call env for the env vars to be set correctly
+  COMMAND="/usr/bin/env $V $VV APP_ROOT=${APP_ROOT} RACK_ENV=${RACK_ENV} RAILS_ENV=${RACK_ENV} $SIDEKIQ -e ${RACK_ENV} -C ${APP_CONFIG}/${CONF_FILE}"
+  
+  if [ ! -d /var/run/engineyard/sidekiq/$APP ]; then
+    mkdir -p /var/run/engineyard/sidekiq/$APP
+  fi
+
+  # handle the second param, don't start if already existing
+
+  logger -t "monit-sidekiq[$$]" "${ACTION}ing Sidekiq worker $WORKER_REF"
+  case "$ACTION" in
+    start)
+      lock
+      cd $APP_ROOT
+      if [ -f $PID_FILE ]; then
+        set_pid_from_file
+        if [ -d /proc/$PID ]; then
+          logger -t "monit-sidekiq[$$]" "Sidekiq worker $WORKER_REF is already running with $PID."
+          RESULT=1
+        else
+          rm -f $PID_FILE
+          logger -t "monit-sidekiq[$$]" "Removing stale pid file ($PID_FILE) for pid $PID"
+        fi
+      fi
+      if [ ! -f $PID_FILE ]; then
+        sudo -u $USER -H $COMMAND  >> $LOG_FILE 2>&1 &
+        RESULT=$?
+        logger -t "monit-sidekiq[$$]" "Started with pid $! and exit $RESULT"
+        echo $! > $PID_FILE
+        sleep .1
+      fi
+      unlock_and_exit_cleanly 
+      ;;
+    stop|quit)
+      legacy_fix
+      lock
+      SIG="INT"
+      [ -z "$GRACE_TIME" ] && GRACE_TIME=60
+      ALLOW_TIMEOUT=1
+      signal_worker
+      [ -e "$LOCK_FILE" ] && rm $LOCK_FILE
+      unlock_and_exit_cleanly  
+      ;;
+    *)
+      usage
+      exit_cleanly      
+      ;;
+  esac
+else
+  echo "/data/$APP/current doesn't exist."
+  usage
+fi
+

data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb

@@ -0,0 +1,28 @@
+---
+<% if @verbose %>
+:verbose: <%= @verbose %>
+<% end %>
+<% if @namespace %>
+:namespace: <%= @namespace %>
+<% end %>
+<% if @server %>
+:server: <%= @server %>
+<% end %>
+<% if @environment %>
+:environment: <%= @environment %>
+<% end %>
+<% if @require %>
+:require: <%= @require %>
+<% end %>
+<% if @pidfile %>
+:pidfile: <%= @pidfile %>
+<% end %>
+<% if @concurrency %>
+:concurrency: <%= @concurrency %>
+<% end %>
+<% if @queues %>
+:queues:
+  <% @queues.each do |name, priority| %>
+  - [<%= name %>, <%= priority %>]
+  <% end %>
+<% end %>

data/examples/monitrc.conf

@@ -0,0 +1,6 @@
+check process sidekiq_myapp
+  with pidfile /path/to/sidekiq.pid
+  start program = "bundle exec sidekiq -C /path/to/sidekiq_conf.yml -P /path/to/sidekiq.pid" with timeout 90 seconds
+  stop program = "kill -s INT `cat /path/to/sidekiq.pid`" with timeout 90 seconds
+  if totalmem is greater than 200 MB for 2 cycles then restart # eating up memory?
+  group myapp_sidekiq

data/examples/por.rb

@@ -1,5 +1,15 @@
 require 'sidekiq'
 
+# If your client is single-threaded, we just need a single connection in our Redis connection pool
+Sidekiq.configure_client do |config|
+  config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :size => 1, :url => 'redis://redis.host:1234/db')
+end
+
+# Sidekiq server is multi-threaded so our Redis connection pool size defaults to concurrency (-c)
+Sidekiq.configure_server do |config|
+  config.redis = Sidekiq::RedisConnection.create(:namespace => 'x', :url => 'redis://redis.host:1234/db')
+end
+
 # Start up sidekiq via
 # ./bin/sidekiq -r ./examples/por.rb
 # and then you can open up an IRB session like so:

data/lib/sidekiq.rb

@@ -2,3 +2,73 @@ require 'sidekiq/version'
 require 'sidekiq/client'
 require 'sidekiq/worker'
 require 'sidekiq/rails' if defined?(::Rails)
+require 'sidekiq/redis_connection'
+
+require 'sidekiq/extensions/action_mailer' if defined?(::ActionMailer)
+require 'sidekiq/extensions/active_record' if defined?(::ActiveRecord)
+
+module Sidekiq
+
+  DEFAULTS = {
+    :queues => [],
+    :concurrency => 25,
+    :require => '.',
+    :environment => nil,
+  }
+
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
+
+  def self.options=(opts)
+    @options = opts
+  end
+
+  ##
+  # Configuration for Sidekiq server, use like:
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.redis = Sidekiq::RedisConnection.create(:namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/mydb')
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #     end
+  #   end
+  def self.configure_server
+    yield self if server?
+  end
+
+  ##
+  # Configuration for Sidekiq client, use like:
+  #
+  #   Sidekiq.configure_client do |config|
+  #     config.redis = Sidekiq::RedisConnection.create(:namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/mydb')
+  #   end
+  def self.configure_client
+    yield self unless server?
+  end
+
+  def self.server?
+    defined?(Sidekiq::CLI)
+  end
+
+  def self.redis
+    @redis ||= Sidekiq::RedisConnection.create
+  end
+
+  def self.redis=(r)
+    @redis = r
+  end
+
+  def self.client_middleware
+    @client_chain ||= Client.default_middleware
+    yield @client_chain if block_given?
+    @client_chain
+  end
+
+  def self.server_middleware
+    @server_chain ||= Processor.default_middleware
+    yield @server_chain if block_given?
+    @server_chain
+  end
+
+end