kulesa-sidekiq 1.2.2

data/config.ru

@@ -0,0 +1,18 @@
+require 'sidekiq'
+
+Sidekiq.configure_client do |config|
+  config.redis = { :size => 1 }
+end
+
+#Sidekiq.redis {|conn| conn.flushdb }
+#10.times do |idx|
+  #Sidekiq::Client.push('class' => 'HardWorker', 'args' => ['foo', 0.1, idx])
+#end
+
+#Sidekiq.redis { |conn| conn.zadd('retry', Time.now.utc.to_f + 3000, MultiJson.encode({
+  #'class' => 'HardWorker', 'args' => ['foo', 0.1, Time.now.to_f],
+  #'queue' => 'default', 'error_message' => 'No such method', 'error_class' => 'NoMethodError',
+  #'failed_at' => Time.now.utc, 'retry_count' => 0 })) }
+
+require 'sidekiq/web'
+run Sidekiq::Web

data/examples/chef/cookbooks/sidekiq/README.rdoc

@@ -0,0 +1,11 @@
+= DESCRIPTION:
+
+Sidekiq is a Redis-backed Ruby library for creating background jobs, placing those jobs on multiple queues, and processing them later.
+
+= USAGE: 
+
+add require_recipe "sidekiq" to main/recipes/default.rb
+
+= NOTES:
+
+I setup a basic size for the Sidekiq workers based on the instance_type, if you need more or less workers please modify the recipe itself.  

data/examples/chef/cookbooks/sidekiq/recipes/default.rb

@@ -0,0 +1,55 @@
+#
+# Cookbook Name:: sidekiq
+# Recipe:: default
+#
+role = node[:instance_role]
+if role == 'solo' || (role == 'util' && node[:name] =~ /sidekiq/)
+
+  # for now
+  worker_count = 1
+
+  node[:applications].each do |app, data|
+    template "/etc/monit.d/sidekiq_#{app}.monitrc" do 
+      owner 'root' 
+      group 'root' 
+      mode 0644 
+      source "monitrc.conf.erb" 
+      variables({ 
+        :num_workers => worker_count,
+        :app_name => app, 
+        :rails_env => node[:environment][:framework_env] 
+      }) 
+    end
+
+    template "/engineyard/bin/sidekiq" do
+      owner 'root'
+      group 'root' 
+      mode 0755
+      source "sidekiq.erb" 
+    end
+
+    worker_count.times do |count|
+      template "/data/#{app}/shared/config/sidekiq_#{count}.yml" do
+        owner node[:owner_name]
+        group node[:owner_name]
+        mode 0644
+        source "sidekiq.yml.erb"
+        variables({
+          :require => "/data/#{app}/current"
+        })
+      end
+    end
+
+    execute "ensure-sidekiq-is-setup-with-monit" do 
+      command %Q{ 
+        monit reload 
+      } 
+    end
+
+    execute "restart-sidekiq" do 
+      command %Q{ 
+        echo "sleep 20 && monit -g #{app}_sidekiq restart all" | at now 
+      }
+    end
+  end 
+end

data/examples/chef/cookbooks/sidekiq/templates/default/monitrc.conf.erb

@@ -0,0 +1,8 @@
+<% (@num_workers || 1).times do |num| %>
+check process sidekiq_<%= @app_name %>_<%= num %>
+  with pidfile /var/run/engineyard/sidekiq/<%= @app_name %>/sidekiq_<%= num %>.pid
+  start program = "/engineyard/bin/sidekiq <%= @app_name %> start <%= @rails_env  %> sidekiq_<%= num %>.yml" with timeout 90 seconds
+  stop program = "/engineyard/bin/sidekiq <%= @app_name %> stop <%= @rails_env  %> sidekiq_<%= num %>.yml" with timeout 90 seconds
+  if totalmem is greater than 300 MB for 2 cycles then restart # eating up memory?
+  group <%= @app_name %>_sidekiq
+<% end %>

data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.erb

@@ -0,0 +1,219 @@
+#!/bin/sh
+#
+# This script starts and stops the Sidekiq daemon
+# This script belongs in /engineyard/bin/sidekiq
+#
+
+PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
+CURDIR=`pwd`
+
+usage() {
+  echo "Usage: $0 <appname> {start|stop|quit} <environment> <conf_file>"
+  echo -e "\nstop)     is a synonym for quit"
+  echo "quit)     issues -INT to request the worker to stop"
+  echo -e "\nSee http://mperham.github.com/sidekiq/ for more details"
+  exit 1
+}
+
+rm_lockfile(){
+  if [ -e $LOCK_FILE ]; then
+   logger -t "monit-sidekiq[$$]" "removing $LOCK_FILE for `cat $LOCK_FILE`"
+   rm $LOCK_FILE
+  fi
+}
+
+exit_cleanly() {
+  cd $CURDIR
+  logger -t "monit-sidekiq[$$]" "exiting wrapper cleanly with $RESULT"
+  exit $RESULT
+}
+
+unlock_and_exit_cleanly(){
+  rm_lockfile
+  exit_cleanly
+}
+
+set_pid_from_file(){
+  export PID=`cat $PID_FILE`
+}
+
+signal_worker() {
+  RESULT=0
+  if [ -f $PID_FILE ]; then
+    set_pid_from_file
+    logger -t "monit-sidekiq[$$]" "Issuing kill with -$SIG $PID"
+    SLEEP_COUNT=0
+    kill -$SIG $PID 
+  fi
+}
+
+signal_worker_fatally(){
+  signal_worker() 
+  if [ -f $PID_FILE ]; then
+    if [ -n "$ALLOW_TIMEOUT" ]; then
+      while [ -e /proc/$PID ]; do
+        sleep .25
+        let "SLEEP_COUNT+=1"
+        let "REPORT_TIME = $SLEEP_COUNT%4"
+        if(( "$SLEEP_COUNT" > $GRACE_TIME )); then
+          logger -t "monit-sidekiq[$$]" "Sidekiq worker with pid $PID for $WORKER_REF still running, issuing -TERM"
+          kill -15 $PID 2>/dev/null; true
+        elif(( $REPORT_TIME == 0 )); then
+          let "RUNTIME = $SLEEP_COUNT/4"
+          logger -t "monit-sidekiq[$$]" "waiting for $PID to die ( for $RUNTIME seconds now)"
+        fi 
+      done
+    fi
+    sleep 1
+    if [ -d /proc/$PID ];then 
+      for child in $(ps axo pid,ppid | awk "{ if ( \$2 == $PID ) { print \$1 }}");
+      do
+        kill -9 $child 2>/dev/null; true
+        logger -t "monit-sidekiq[$$]" "Murdering Sidekiq workers child with $child for $WORKER_REF"
+        break
+      done    
+      while [ -d /proc/$PID ]; do
+        logger -t "monit-sidekiq[$$]" "Murdering Sidekiq worker with $PID for $WORKER_REF"
+        kill -9 $PID
+        sleep 1
+      done
+    fi
+    logger -t "monit-sidekiq[$$]" "Removing pid file for $PID - $WORKER_REF"
+    [ -e "$PID_FILE" -a ! -d /proc/$PID ] && rm -f $PID_FILE
+  fi
+}
+
+lock(){
+  RESULT=0
+  if [ -e $LOCK_FILE ]; then
+    LAST_LOCK_PID=`cat $LOCK_FILE`
+    if [ -n $LAST_LOCK_PID -a -z "`ps axo pid|grep $LAST_LOCK_PID`" -a -f $LOCK_FILE ];then
+      sleep 1
+      logger -t "monit-sidekiq[$$]" "Removing stale lock file for $WORKER_REF ($LAST_LOCK_PID)"
+      rm $LOCK_FILE 2>&1
+    else
+      logger -t "monit-sidekiq[$$]" "Monit already messing with $WORKER_REF ($LAST_LOCK_PID)"
+      RESULT=1
+      exit_cleanly 
+    fi
+  fi
+  echo $$ > $LOCK_FILE
+}
+
+legacy_fix() {
+ #In the transition from 0.18.2 to 0.18.3 of ey monit scripts the way 
+ #the pid file is used to find the process to kill has changed. 
+ #To avert problems being left behind after an upgrade of this package,
+  if [ -f $PID_FILE ]; then
+    set_pid_from_file
+    if [ -n "`ps axo pid,command|grep $PID|grep 'su -c'`" ];then
+      logger -t "monit-sidekiq[$$]" "Monit Scripts have just been upgraded, killing old style workers"
+      for child in $(ps axo pid,ppid| awk "{ if ( \$2 == $PID ) { print \$1 }}");
+      do
+        kill -TERM $child 2> /dev/null
+        while [ -e /proc/$child ]; do
+          logger -t "monit-sidekiq[$$]" "killing legacy worker: $child"
+          [ -e /proc/$child ] && kill -9 $child 2> /dev/null
+	  sleep 1 
+        done
+      done
+      [ -e /proc/$PID   ] && kill -9 $PID   2> /dev/null
+      rm $PID_FILE
+      unlock_exit_cleanly  
+    fi
+  fi
+}
+
+if [ $# -lt 4 ]; then usage; fi
+
+if [ "`whoami`" != "root" ]; then
+  logger -t `basename $0` -s "Must be run as root" 
+  exit 1
+fi
+
+#Baisc Setup of default values
+APP=$1 ; ACTION=$2; RACK_ENV=$3; CONF_FILE=$4;
+
+APP_DIR="/data/${APP}"
+APP_ROOT="${APP_DIR}/current"
+APP_SHARED="${APP_DIR}/shared"
+APP_CONFIG="${APP_SHARED}/config"
+
+if [ -e "${APP_CONFIG}/${CONF_FILE}" ]; then
+  logger -t "sidekiq_${APP}" -s "Good, found a conf file. Proceeding..."
+else
+  logger -t "sidekiq_${APP}" -s "/data/${APP}/shared/config/${CONF_FILE} not found for app: ${APP}"
+  exit 1
+fi
+
+WORKER_REF=`echo $CONF_FILE | sed s/.conf//`
+LOG_FILE="$APP_ROOT/log/$WORKER_REF.log"
+LOCK_FILE="/tmp/$WORKER_REF.monit-lock"
+PID_FILE="/var/run/engineyard/sidekiq/$APP/$WORKER_REF.pid"
+GEMFILE="$APP_ROOT/Gemfile"
+SIDEKIQ="sidekiq"
+if [ -f $GEMFILE ];then
+  SIDEKIQ="bundle exec $APP_ROOT/ey_bundler_binstubs/sidekiq"
+fi
+
+if [ -d $APP_ROOT ]; then
+  USER=$(stat -L -c"%U" $APP_ROOT)
+  export HOME="/home/$USER" 
+
+  # Fix for SD-3786 - stop sending in VERBOSE= and VVERBOSE= by default
+  if declare -p VERBOSE >/dev/null 2>&1; then export V="VERBOSE=$VERBOSE"; fi
+  if declare -p VVERBOSE >/dev/null 2>&1; then export VV="VVERBOSE=$VVERBOSE"; fi
+
+  # Older versions of sudo need us to call env for the env vars to be set correctly
+  COMMAND="/usr/bin/env $V $VV APP_ROOT=${APP_ROOT} RACK_ENV=${RACK_ENV} RAILS_ENV=${RACK_ENV} $SIDEKIQ -e ${RACK_ENV} -C ${APP_CONFIG}/${CONF_FILE}"
+  
+  if [ ! -d /var/run/engineyard/sidekiq/$APP ]; then
+    mkdir -p /var/run/engineyard/sidekiq/$APP
+  fi
+
+  # handle the second param, don't start if already existing
+
+  logger -t "monit-sidekiq[$$]" "${ACTION}ing Sidekiq worker $WORKER_REF"
+  case "$ACTION" in
+    start)
+      lock
+      cd $APP_ROOT
+      if [ -f $PID_FILE ]; then
+        set_pid_from_file
+        if [ -d /proc/$PID ]; then
+          logger -t "monit-sidekiq[$$]" "Sidekiq worker $WORKER_REF is already running with $PID."
+          RESULT=1
+        else
+          rm -f $PID_FILE
+          logger -t "monit-sidekiq[$$]" "Removing stale pid file ($PID_FILE) for pid $PID"
+        fi
+      fi
+      if [ ! -f $PID_FILE ]; then
+        sudo -u $USER -H $COMMAND  >> $LOG_FILE 2>&1 &
+        RESULT=$?
+        logger -t "monit-sidekiq[$$]" "Started with pid $! and exit $RESULT"
+        echo $! > $PID_FILE
+        sleep .1
+      fi
+      unlock_and_exit_cleanly 
+      ;;
+    stop|quit)
+      legacy_fix
+      lock
+      SIG="INT"
+      [ -z "$GRACE_TIME" ] && GRACE_TIME=60
+      ALLOW_TIMEOUT=1
+      signal_worker
+      [ -e "$LOCK_FILE" ] && rm $LOCK_FILE
+      unlock_and_exit_cleanly  
+      ;;
+    *)
+      usage
+      exit_cleanly      
+      ;;
+  esac
+else
+  echo "/data/$APP/current doesn't exist."
+  usage
+fi
+

data/examples/chef/cookbooks/sidekiq/templates/default/sidekiq.yml.erb

@@ -0,0 +1,22 @@
+---
+<% if @verbose %>
+:verbose: <%= @verbose %>
+<% end %>
+<% if @environment %>
+:environment: <%= @environment %>
+<% end %>
+<% if @require %>
+:require: <%= @require %>
+<% end %>
+<% if @pidfile %>
+:pidfile: <%= @pidfile %>
+<% end %>
+<% if @concurrency %>
+:concurrency: <%= @concurrency %>
+<% end %>
+<% if @queues %>
+:queues:
+  <% @queues.each do |name, priority| %>
+  - [<%= name %>, <%= priority %>]
+  <% end %>
+<% end %>

data/examples/clockwork.rb

@@ -0,0 +1,44 @@
+# Sidekiq defers scheduling to other, better suited gems.
+# If you want to run a job regularly, here's an example
+# of using the 'clockwork' gem to push jobs to Sidekiq
+# regularly.
+
+# require boot & environment for a Rails app
+# require_relative "../config/boot"
+# require_relative "../config/environment"
+require "clockwork"
+
+class MyWorker
+  include Sidekiq::Worker
+
+  def perform(count)
+    puts "Job ##{count}: Late night, so tired..."
+  end
+
+  def self.late_night_work
+    10.times do |x|
+      perform_async(x)
+    end
+  end
+end
+
+class HourlyWorker
+  include Sidekiq::Worker
+
+  def perform
+    cleanup_database
+    format_hard_drive
+  end
+end
+
+module Clockwork
+  # Kick off a bunch of jobs early in the morning
+  every 1.day, 'my_worker.late_night_work', :at => '4:30 am' do
+    MyWorker.late_night_work
+  end
+
+  every 1.hour do
+    HourlyWorker.perform_async
+  end
+end
+

data/examples/config.yml

@@ -0,0 +1,10 @@
+# Sample configuration file for Sidekiq.
+# Options here can still be overridden by cmd line args.
+#   sidekiq -C config.yml
+---
+:verbose: false
+:concurrency:  25
+:queues:
+  - [often, 7]
+  - [default, 5]
+  - [seldom, 3]

data/examples/monitrc.conf

@@ -0,0 +1,6 @@
+check process sidekiq_myapp
+  with pidfile /path/to/sidekiq.pid
+  start program = "bundle exec sidekiq -C /path/to/sidekiq_conf.yml -P /path/to/sidekiq.pid" with timeout 90 seconds
+  stop program = "kill -s INT `cat /path/to/sidekiq.pid`" with timeout 90 seconds
+  if totalmem is greater than 200 MB for 2 cycles then restart # eating up memory?
+  group myapp_sidekiq

data/examples/por.rb

@@ -0,0 +1,27 @@
+require 'sidekiq'
+
+# If your client is single-threaded, we just need a single connection in our Redis connection pool
+Sidekiq.configure_client do |config|
+  config.redis = { :namespace => 'x', :size => 1, :url => 'redis://redis.host:1234/14' }
+end
+
+# Sidekiq server is multi-threaded so our Redis connection pool size defaults to concurrency (-c)
+Sidekiq.configure_server do |config|
+  config.redis = { :namespace => 'x', :url => 'redis://redis.host:1234/14' }
+end
+
+# Start up sidekiq via
+# ./bin/sidekiq -r ./examples/por.rb
+# and then you can open up an IRB session like so:
+# irb -r ./examples/por.rb
+# where you can then say
+# PlainOldRuby.perform_async "like a dog", 3
+#
+class PlainOldRuby
+  include Sidekiq::Worker
+
+  def perform(how_hard="super hard", how_long=1)
+    sleep how_long
+    puts "Workin' #{how_hard}"
+  end
+end

data/examples/scheduling.rb

@@ -0,0 +1,37 @@
+# Sidekiq defers scheduling to other, better suited gems.
+# If you want to run a job regularly, here's an example
+# of using the 'whenever' gem to push jobs to Sidekiq
+# regularly.
+
+class MyWorker
+  include Sidekiq::Worker
+
+  def perform(count)
+    puts "Job ##{count}: Late night, so tired..."
+  end
+
+  def self.late_night_work
+    10.times do |x|
+      perform_async(x)
+    end
+  end
+end
+
+# Kick off a bunch of jobs early in the morning
+every 1.day, :at => '4:30 am' do
+  runner "MyWorker.late_night_work"
+end
+
+
+class HourlyWorker
+  include Sidekiq::Worker
+
+  def perform
+    cleanup_database
+    format_hard_drive
+  end
+end
+
+every :hour do # Many shortcuts available: :hour, :day, :month, :year, :reboot
+  runner "HourlyWorker.perform_async"
+end