eycloud-recipe-sidekiq 0.1.0 → 0.2.0

data/README.md

@@ -1,10 +1,14 @@
 # Sidekiq recipe for EY Cloud
 
-DESCRIPTION HERE
+Sidekiq is a Redis-backed Ruby library for creating background jobs, placing those jobs on multiple queues, and processing them later.
 
 ## Installation
 
-INSTALLATION HERE
+Add the following to `main/recipes/default.rb` in your cookbooks:
+
+```ruby
+require_recipe "sidekiq"
+```
 
 ## Contributing
 

data/eycloud-recipe-sidekiq.gemspec

@@ -1,12 +1,12 @@
 # -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 
-version = "0.1.0" # TODO get from metadata.json or .rb
+version = "0.2.0" # TODO get from metadata.json or .rb
 
 Gem::Specification.new do |s|
   s.name        = "eycloud-recipe-sidekiq"
   s.version     = version
-  s.authors     = ["Dr Nic Williams"]
+  s.authors     = ["Dr Nic Williams", "Jesse Cooke"]
   s.email       = ["drnicwilliams@gmail.com"]
   s.homepage    = ""
   s.summary     = %q{Sidekiq recipe for EY Cloud} # TODO from metadata
@@ -17,6 +17,6 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  # s.add_dependency("eycloud-helper-cronjobs")
+  # s.add_dependency("eycloud-helper-common")
   s.add_development_dependency("rake")
 end

data/libraries/sidekiq_instances.rb

@@ -0,0 +1,9 @@
+class Chef
+  class Recipe
+    # Does this instance run sidekiq?
+    def sidekiq_instance?
+      role = node[:instance_role]
+      role == 'solo' || role == 'eylocal' || (role == 'util' && node[:name] =~ /^sidekiq/)
+    end
+  end
+end

data/metadata.json

@@ -5,7 +5,7 @@
   "license": "MIT",
   "maintainer": "Dr Nic Williams",
   "maintainer_email": "drnicwilliams@gmail.com",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "attributes": {
   },
   "dependencies": {

data/metadata.rb

@@ -3,5 +3,5 @@ description       "Sidekiq recipe for EY Cloud"
 long_description  IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 maintainer        "Dr Nic Williams"
 maintainer_email  "drnicwilliams@gmail.com"
-version           "0.1.0"
+version           "0.2.0"
 # depends           "cronjobs"

data/recipes/configure.rb

@@ -1,22 +1,39 @@
-#
-# Cookbook Name:: sidekiq
-# Recipe:: restart
-#
-
 if sidekiq_instance?
 
-  node[:applications].each do |app_name, data|
-    
-    template "/etc/monit.d/sidekiq_#{app_name}.monitrc" do 
-      owner 'root'
-      group 'root'
-      mode 0644
-      source "monitrc.erb" 
+  # for now
+  worker_count = 1
+
+  node[:applications].each do |app, data|
+    template "/etc/monit.d/sidekiq_#{app}.monitrc" do 
+      owner 'root' 
+      group 'root' 
+      mode 0644 
+      source "monitrc.conf.erb" 
       variables({ 
-        :command_name => "sidekiq",
-        :app_name => app_name
+        :num_workers => worker_count,
+        :app_name => app, 
+        :rails_env => node[:environment][:framework_env] 
       }) 
     end
-    
-  end
-end
+
+    template "/engineyard/bin/sidekiq" do
+      owner 'root'
+      group 'root' 
+      mode 0755
+      source "sidekiq.erb" 
+    end
+
+    worker_count.times do |count|
+      template "/data/#{app}/shared/config/sidekiq_#{count}.yml" do
+        owner node[:owner_name]
+        group node[:owner_name]
+        mode 0644
+        source "sidekiq.yml.erb"
+        variables({
+          :require => "/data/#{app}/current"
+        })
+      end
+    end
+
+  end 
+end

data/recipes/default.rb

@@ -4,4 +4,4 @@
 #
 
 require_recipe "sidekiq::configure"
-require_recipe "sidekiq::restart"
+require_recipe "sidekiq::restart"

data/recipes/restart.rb

@@ -1,7 +1,16 @@
-#
-# Cookbook Name:: sidekiq
-# Recipe:: restart
-#
 
 if sidekiq_instance?
+
+  execute "ensure-sidekiq-is-setup-with-monit" do 
+    command %Q{ 
+      monit reload 
+    } 
+  end
+
+  execute "restart-sidekiq" do 
+    command %Q{ 
+      echo "sleep 20 && monit -g #{app}_sidekiq restart all" | at now 
+    }
+  end
+
 end

data/templates/default/monitrc.conf.erb

@@ -0,0 +1,8 @@
+<% (@num_workers || 1).times do |num| %>
+check process sidekiq_<%= @app_name %>_<%= num %>
+  with pidfile /var/run/engineyard/sidekiq/<%= @app_name %>/sidekiq_<%= num %>.pid
+  start program = "/engineyard/bin/sidekiq <%= @app_name %> start <%= @rails_env  %> sidekiq_<%= num %>.yml" with timeout 90 seconds
+  stop program = "/engineyard/bin/sidekiq <%= @app_name %> stop <%= @rails_env  %> sidekiq_<%= num %>.yml" with timeout 90 seconds
+  if totalmem is greater than 300 MB for 2 cycles then restart # eating up memory?
+  group <%= @app_name %>_sidekiq
+<% end %>

data/templates/default/sidekiq.erb

@@ -0,0 +1,219 @@
+#!/bin/sh
+#
+# This script starts and stops the Sidekiq daemon
+# This script belongs in /engineyard/bin/sidekiq
+#
+
+PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
+CURDIR=`pwd`
+
+usage() {
+  echo "Usage: $0 <appname> {start|stop|quit} <environment> <conf_file>"
+  echo -e "\nstop)     is a synonym for quit"
+  echo "quit)     issues -INT to request the worker to stop"
+  echo -e "\nSee http://mperham.github.com/sidekiq/ for more details"
+  exit 1
+}
+
+rm_lockfile(){
+  if [ -e $LOCK_FILE ]; then
+   logger -t "monit-sidekiq[$$]" "removing $LOCK_FILE for `cat $LOCK_FILE`"
+   rm $LOCK_FILE
+  fi
+}
+
+exit_cleanly() {
+  cd $CURDIR
+  logger -t "monit-sidekiq[$$]" "exiting wrapper cleanly with $RESULT"
+  exit $RESULT
+}
+
+unlock_and_exit_cleanly(){
+  rm_lockfile
+  exit_cleanly
+}
+
+set_pid_from_file(){
+  export PID=`cat $PID_FILE`
+}
+
+signal_worker() {
+  RESULT=0
+  if [ -f $PID_FILE ]; then
+    set_pid_from_file
+    logger -t "monit-sidekiq[$$]" "Issuing kill with -$SIG $PID"
+    SLEEP_COUNT=0
+    kill -$SIG $PID 
+  fi
+}
+
+signal_worker_fatally(){
+  signal_worker() 
+  if [ -f $PID_FILE ]; then
+    if [ -n "$ALLOW_TIMEOUT" ]; then
+      while [ -e /proc/$PID ]; do
+        sleep .25
+        let "SLEEP_COUNT+=1"
+        let "REPORT_TIME = $SLEEP_COUNT%4"
+        if(( "$SLEEP_COUNT" > $GRACE_TIME )); then
+          logger -t "monit-sidekiq[$$]" "Sidekiq worker with pid $PID for $WORKER_REF still running, issuing -TERM"
+          kill -15 $PID 2>/dev/null; true
+        elif(( $REPORT_TIME == 0 )); then
+          let "RUNTIME = $SLEEP_COUNT/4"
+          logger -t "monit-sidekiq[$$]" "waiting for $PID to die ( for $RUNTIME seconds now)"
+        fi 
+      done
+    fi
+    sleep 1
+    if [ -d /proc/$PID ];then 
+      for child in $(ps axo pid,ppid | awk "{ if ( \$2 == $PID ) { print \$1 }}");
+      do
+        kill -9 $child 2>/dev/null; true
+        logger -t "monit-sidekiq[$$]" "Murdering Sidekiq workers child with $child for $WORKER_REF"
+        break
+      done    
+      while [ -d /proc/$PID ]; do
+        logger -t "monit-sidekiq[$$]" "Murdering Sidekiq worker with $PID for $WORKER_REF"
+        kill -9 $PID
+        sleep 1
+      done
+    fi
+    logger -t "monit-sidekiq[$$]" "Removing pid file for $PID - $WORKER_REF"
+    [ -e "$PID_FILE" -a ! -d /proc/$PID ] && rm -f $PID_FILE
+  fi
+}
+
+lock(){
+  RESULT=0
+  if [ -e $LOCK_FILE ]; then
+    LAST_LOCK_PID=`cat $LOCK_FILE`
+    if [ -n $LAST_LOCK_PID -a -z "`ps axo pid|grep $LAST_LOCK_PID`" -a -f $LOCK_FILE ];then
+      sleep 1
+      logger -t "monit-sidekiq[$$]" "Removing stale lock file for $WORKER_REF ($LAST_LOCK_PID)"
+      rm $LOCK_FILE 2>&1
+    else
+      logger -t "monit-sidekiq[$$]" "Monit already messing with $WORKER_REF ($LAST_LOCK_PID)"
+      RESULT=1
+      exit_cleanly 
+    fi
+  fi
+  echo $$ > $LOCK_FILE
+}
+
+legacy_fix() {
+ #In the transition from 0.18.2 to 0.18.3 of ey monit scripts the way 
+ #the pid file is used to find the process to kill has changed. 
+ #To avert problems being left behind after an upgrade of this package,
+  if [ -f $PID_FILE ]; then
+    set_pid_from_file
+    if [ -n "`ps axo pid,command|grep $PID|grep 'su -c'`" ];then
+      logger -t "monit-sidekiq[$$]" "Monit Scripts have just been upgraded, killing old style workers"
+      for child in $(ps axo pid,ppid| awk "{ if ( \$2 == $PID ) { print \$1 }}");
+      do
+        kill -TERM $child 2> /dev/null
+        while [ -e /proc/$child ]; do
+          logger -t "monit-sidekiq[$$]" "killing legacy worker: $child"
+          [ -e /proc/$child ] && kill -9 $child 2> /dev/null
+	  sleep 1 
+        done
+      done
+      [ -e /proc/$PID   ] && kill -9 $PID   2> /dev/null
+      rm $PID_FILE
+      unlock_exit_cleanly  
+    fi
+  fi
+}
+
+if [ $# -lt 4 ]; then usage; fi
+
+if [ "`whoami`" != "root" ]; then
+  logger -t `basename $0` -s "Must be run as root" 
+  exit 1
+fi
+
+#Baisc Setup of default values
+APP=$1 ; ACTION=$2; RACK_ENV=$3; CONF_FILE=$4;
+
+APP_DIR="/data/${APP}"
+APP_ROOT="${APP_DIR}/current"
+APP_SHARED="${APP_DIR}/shared"
+APP_CONFIG="${APP_SHARED}/config"
+
+if [ -e "${APP_CONFIG}/${CONF_FILE}" ]; then
+  logger -t "sidekiq_${APP}" -s "Good, found a conf file. Proceeding..."
+else
+  logger -t "sidekiq_${APP}" -s "/data/${APP}/shared/config/${CONF_FILE} not found for app: ${APP}"
+  exit 1
+fi
+
+WORKER_REF=`echo $CONF_FILE | sed s/.conf//`
+LOG_FILE="$APP_ROOT/log/$WORKER_REF.log"
+LOCK_FILE="/tmp/$WORKER_REF.monit-lock"
+PID_FILE="/var/run/engineyard/sidekiq/$APP/$WORKER_REF.pid"
+GEMFILE="$APP_ROOT/Gemfile"
+SIDEKIQ="sidekiq"
+if [ -f $GEMFILE ];then
+  SIDEKIQ="bundle exec $APP_ROOT/ey_bundler_binstubs/sidekiq"
+fi
+
+if [ -d $APP_ROOT ]; then
+  USER=$(stat -L -c"%U" $APP_ROOT)
+  export HOME="/home/$USER" 
+
+  # Fix for SD-3786 - stop sending in VERBOSE= and VVERBOSE= by default
+  if declare -p VERBOSE >/dev/null 2>&1; then export V="VERBOSE=$VERBOSE"; fi
+  if declare -p VVERBOSE >/dev/null 2>&1; then export VV="VVERBOSE=$VVERBOSE"; fi
+
+  # Older versions of sudo need us to call env for the env vars to be set correctly
+  COMMAND="/usr/bin/env $V $VV APP_ROOT=${APP_ROOT} RACK_ENV=${RACK_ENV} RAILS_ENV=${RACK_ENV} $SIDEKIQ -e ${RACK_ENV} -C ${APP_CONFIG}/${CONF_FILE}"
+  
+  if [ ! -d /var/run/engineyard/sidekiq/$APP ]; then
+    mkdir -p /var/run/engineyard/sidekiq/$APP
+  fi
+
+  # handle the second param, don't start if already existing
+
+  logger -t "monit-sidekiq[$$]" "${ACTION}ing Sidekiq worker $WORKER_REF"
+  case "$ACTION" in
+    start)
+      lock
+      cd $APP_ROOT
+      if [ -f $PID_FILE ]; then
+        set_pid_from_file
+        if [ -d /proc/$PID ]; then
+          logger -t "monit-sidekiq[$$]" "Sidekiq worker $WORKER_REF is already running with $PID."
+          RESULT=1
+        else
+          rm -f $PID_FILE
+          logger -t "monit-sidekiq[$$]" "Removing stale pid file ($PID_FILE) for pid $PID"
+        fi
+      fi
+      if [ ! -f $PID_FILE ]; then
+        sudo -u $USER -H $COMMAND  >> $LOG_FILE 2>&1 &
+        RESULT=$?
+        logger -t "monit-sidekiq[$$]" "Started with pid $! and exit $RESULT"
+        echo $! > $PID_FILE
+        sleep .1
+      fi
+      unlock_and_exit_cleanly 
+      ;;
+    stop|quit)
+      legacy_fix
+      lock
+      SIG="INT"
+      [ -z "$GRACE_TIME" ] && GRACE_TIME=60
+      ALLOW_TIMEOUT=1
+      signal_worker
+      [ -e "$LOCK_FILE" ] && rm $LOCK_FILE
+      unlock_and_exit_cleanly  
+      ;;
+    *)
+      usage
+      exit_cleanly      
+      ;;
+  esac
+else
+  echo "/data/$APP/current doesn't exist."
+  usage
+fi
+

data/templates/default/sidekiq.yml.erb

@@ -0,0 +1,22 @@
+---
+<% if @verbose %>
+:verbose: <%= @verbose %>
+<% end %>
+<% if @environment %>
+:environment: <%= @environment %>
+<% end %>
+<% if @require %>
+:require: <%= @require %>
+<% end %>
+<% if @pidfile %>
+:pidfile: <%= @pidfile %>
+<% end %>
+<% if @concurrency %>
+:concurrency: <%= @concurrency %>
+<% end %>
+<% if @queues %>
+:queues:
+  <% @queues.each do |name, priority| %>
+  - [<%= name %>, <%= priority %>]
+  <% end %>
+<% end %>

metadata

@@ -1,19 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: eycloud-recipe-sidekiq
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
 - Dr Nic Williams
+- Jesse Cooke
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-20 00:00:00.000000000 Z
+date: 2012-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70160749094480 !ruby/object:Gem::Requirement
+  requirement: &70286679065560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +22,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70160749094480
+  version_requirements: *70286679065560
 description: Sidekiq recipe for EY Cloud
 email:
 - drnicwilliams@gmail.com
@@ -35,15 +36,15 @@ files:
 - README.md
 - Rakefile
 - eycloud-recipe-sidekiq.gemspec
-- libraries/find_sidekiq_instances.rb
+- libraries/sidekiq_instances.rb
 - metadata.json
 - metadata.rb
 - recipes/configure.rb
 - recipes/default.rb
-- recipes/install.rb
 - recipes/restart.rb
-- template/default/monitrc.erb
-- template/default/sidekiq.erb
+- templates/default/monitrc.conf.erb
+- templates/default/sidekiq.erb
+- templates/default/sidekiq.yml.erb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -58,7 +59,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3628996913867977882
+      hash: 3781183210708579483
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -67,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3628996913867977882
+      hash: 3781183210708579483
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.17

data/libraries/find_sidekiq_instances.rb

@@ -1,21 +0,0 @@
-class Chef
-  class Recipe
-    # Return which instance is to have redis installed on it
-    # This is determind as follows:
-    #   1. A utility prefixed with 'resque'
-    #   2. A solo
-    # Returns array of hash of instance data, including { "id" => "i-123456", }
-    def sidekiq_instances
-      @resque_instances ||= node[:engineyard][:environment][:instances].find do |x|
-        x[:role] == "solo" || node[:instance_role] == "eylocal" || 
-                        (node[:role] == "util" && node[:name] == "sidekiq"
-      end
-    end
-    
-    # Does this instance run resque?
-    def sidekiq_instance?
-      node[:instance_role] == "solo" || node[:instance_role] == "eylocal" || 
-              (node[:instance_role] == "util" && node[:name] == "sidekiq"
-    end
-  end
-end

data/recipes/install.rb

@@ -1,19 +0,0 @@
-#
-# Cookbook Name:: sidekiq
-# Recipe:: install
-#
-
-if sidekiq_instance?
-
-  template "/engineyard/bin/sidekiq" do
-    source "sidekiq.erb"
-    owner "root"
-    group "root"
-    mode 0655
-    variables({
-      :command_name => "sidekiq",
-      :command => "bundle exec sidekiq"
-    })
-  end
-
-end

data/template/default/monitrc.erb

@@ -1,7 +0,0 @@
-check process <%= "#{@app_name}_#{@command_name}" %>
-  with pidfile /var/run/engineyard/<%= @command_name %>/<%= @app_name %>/<%=@command_name %>.pid
-  start program = "/engineyard/bin/<%= "#{@command_name}" %>_wrapper <%= @app_name %> start"
-  stop program = "/engineyard/bin/<%= "#{@command_name}" %>_wrapper <%= @app_name %> stop"
-  # if mem > 200 MB for 2 cycles then restart
-  # if cpu > 100% for 2 cycles then restart
-  group <%= "#{@app_name}_#{@command_name}" %>

data/template/default/sidekiq.erb

@@ -1,78 +0,0 @@
-#!/bin/sh
-#
-# This script starts and stops the Sidekiq daemon
-# This script belongs in /engineyard/bin/sidekiq
-#
-# Run the script as follows:
-# /engineyard/bin/sidekiq app_name start|stop
-#
-
-# These variables are managed by Chef - modify them in recipes/default.rb
-COMMAND_NAME="<%= @command_name %>" # Alpha-numeric characters and underscores only
-COMMAND="sidekiq"
-
-CURDIR=`pwd`
-PATH=/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$PATH
-
-RAILS_ROOT=/data/$1/current
-
-PID_FILE=/var/run/engineyard/$COMMAND_NAME/$1/$COMMAND_NAME.pid
-EXEC_LOG_FILE=/var/log/engineyard/$COMMAND_NAME/wrapper_$1.log
-
-usage() {
-  echo "Usage: $0 <appname> {start|stop}"
-  exit 1
-}
-
-if [ $# -lt 2 ]; then usage; fi
-
-if [ -d $RAILS_ROOT ]; then
-  mkdir -p /var/log/engineyard/$COMMAND_NAME/$1
-  mkdir -p /var/run/engineyard/$COMMAND_NAME/$1
-  
-  # handle the second param, don't start if already existing
-  case "$2" in
-    start)
-      cd $RAILS_ROOT
-      echo "Starting $COMMAND_NAME"
-      if [ -f $PID_FILE ]; then
-        PID=`cat $PID_FILE`
-        if [ -d /proc/$PID ]; then
-          echo "$COMMAND_NAME is already running."
-          exit 1
-        fi
-        rm -f $PID_FILE
-      fi
-      exec $COMMAND 1>$EXEC_LOG_FILE 2>&1 <%= "&" if @run_in_background %> 
-      <%= @run_in_background ? "echo $! > $PID_FILE;" : "echo $$ > $PID_FILE" %>
-      ;;
-    stop)
-      GRACE_TIME=${GRACE_TIME:-40}
-      echo "Stopping $COMMAND_NAME"
-      if [ -f $PID_FILE ]; then
-        PID=`cat $PID_FILE`
-        kill $PID 2>/dev/null; true
-        SLEEP_COUNT=0
-        while [ -e /proc/$PID ]; do
-          sleep .25
-          let "SLEEP_COUNT+=1"
-          if(( "$SLEEP_COUNT" > $GRACE_TIME )); then
-            echo "Stop wait time exceeded. Killing..."
-            kill -9 $PID 2>/dev/null; true
-            break
-          fi
-        done
-        rm $PID_FILE
-      else
-        echo "No PID file found! Cannot stop $COMMAND_NAME"
-      fi
-      ;;
-    *)
-      usage
-      ;;
-        esac
-else
-  echo "$RAILS_ROOT doesn't exist."
-  usage
-fi
-cd $CURDIR