sweatshop 1.4.0

data/lib/sweat_shop/worker.rb

@@ -0,0 +1,162 @@
+require File.dirname(__FILE__) + '/metaid'
+
+module SweatShop
+  class Worker
+    def self.inherited(subclass)
+      self.workers << subclass
+    end
+
+    def self.method_missing(method, *args, &block)
+      if method.to_s =~ /^async_(.*)/
+        method        = $1
+        expected_args = instance.method(method).arity
+        if expected_args != args.size
+          raise ArgumentError.new("#{method} expects #{expected_args} arguments")
+        end
+        return instance.send(method, *args) unless config['enable']
+
+        uid  = ::Digest::MD5.hexdigest("#{name}:#{method}:#{args}:#{Time.now.to_f}")
+        task = {:args => args, :method => method, :uid => uid, :queued_at => Time.now.to_i}
+
+        log("Putting #{uid} on #{queue_name}")
+        enqueue(task)
+
+        uid
+      elsif instance.respond_to?(method)
+        instance.send(method, *args)
+      else
+        super
+      end
+    end
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def self.config
+      SweatShop.config
+    end
+
+    def self.queue_name
+      @queue_name ||= self.to_s
+    end
+
+    def self.delete_queue
+      queue.delete(queue_name)
+    end
+
+    def self.queue_size
+      queue.queue_size(queue_name)
+    end
+
+    def self.enqueue(task)
+      queue.enqueue(queue_name, task)
+    end
+
+    def self.dequeue
+      queue.dequeue(queue_name)
+    end
+
+    def self.confirm
+      queue.confirm(queue_name)
+    end
+
+    def self.subscribe
+      queue.subscribe(queue_name) do |task|
+        do_task(task)
+      end
+    end
+
+    def self.do_tasks
+      while task = dequeue
+        do_task(task)
+      end
+    end
+
+    def self.do_task(task)
+      begin
+        call_before_task(task)
+
+        queued_at = task[:queued_at] ? "(queued #{Time.at(task[:queued_at]).strftime('%Y/%m/%d %H:%M:%S')})" : ''
+        log("Dequeuing #{queue_name}::#{task[:method]} #{queued_at}")
+        task[:result] = instance.send(task[:method], *task[:args])
+
+        call_after_task(task)
+        confirm
+      rescue SystemExit
+        exit
+      rescue Exception => e
+        log("Caught Exception: #{e.message}, \n#{e.backtrace.join("\n")}")
+        call_exception_handler(e)
+      end
+    end
+
+    def self.queue
+      SweatShop.queue(queue_group.to_s)
+    end
+
+    def self.workers
+      SweatShop.workers
+    end
+
+    def self.config
+      SweatShop.config
+    end
+
+    def self.log(msg)
+      SweatShop.log(msg)
+    end
+
+    def self.call_before_task(task)
+      superclass.call_before_task(task) if superclass.respond_to?(:call_before_task)
+      before_task.call(task) if before_task
+    end
+
+    def self.call_after_task(task)
+      superclass.call_after_task(task) if superclass.respond_to?(:call_after_task)
+      after_task.call(task) if after_task
+    end
+
+    def self.call_exception_handler(exception)
+      superclass.call_exception_handler(exception) if superclass.respond_to?(:call_exception_handler)
+      on_exception.call(exception) if on_exception
+    end
+
+    def self.before_task(&block)
+      if block
+        @before_task = block
+      else
+        @before_task
+      end
+    end
+
+    def self.after_task(&block)
+      if block
+        @after_task = block
+      else
+        @after_task
+      end
+    end
+
+    def self.on_exception(&block)
+      if block
+        @on_exception = block
+      else
+        @on_exception
+      end
+    end
+
+    def self.stop
+      instance.stop
+    end
+
+    # called before we exit -- subclass can implement this method
+    def stop; end;
+
+
+    def self.queue_group(group=nil)
+      group ? meta_def(:_queue_group){ group } : _queue_group
+    end
+    queue_group :default
+  end
+end

data/script/initd.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# chkconfig: 345 80 20
+# description: Sweatshop starts the sweatd daemon
+
+# Source function library.
+if [ -f /etc/init.d/functions ] ; then
+	. /etc/init.d/functions
+elif [ -f /etc/rc.d/init.d/functions ] ; then
+	. /etc/rc.d/init.d/functions
+else
+	exit 0
+fi
+
+DAEMON=/opt/current/script/sweatshop
+NAME=sweatshop
+PID_DIR=/opt/current/log
+LOG_FILE=$PID_DIR/sweatshop.log
+INSTANCES=3
+#QUEUE_GROUPS=test
+export RAILS_ENV=dev
+
+[ -f /etc/sysconfig/${NAME} ] && . /etc/sysconfig/${NAME}
+
+
+# Gracefully exit if the package has been removed.
+if [ ! -x $DAEMON ]; then
+  echo "$DAEMON does not exist"
+  exit 0
+fi;
+
+
+function start() {
+  num=$1
+  pidfile=$PID_DIR/$NAME.$num.pid
+  echo $"Starting ${NAME}:${num} "
+  if [ -z $QUEUE_GROUPS ]; then
+    $DAEMON -d start --log-file $LOG_FILE --pid-file $pidfile 
+  else
+    $DAEMON -d start --log-file $LOG_FILE --pid-file $pidfile --groups $QUEUE_GROUPS
+  fi
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && success || failure
+  echo
+  return $RETVAL
+}
+
+function stop() {
+  num=$1
+  pidfile=$PID_DIR/$NAME.$num.pid
+  echo $"Stopping ${NAME}:${num}: "
+  $DAEMON -d stop --log-file $LOG_FILE --pid-file $pidfile
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && success || failure
+  echo
+  return $RETVAL
+}
+
+function reload() {
+  num=$1
+  pidfile=$PID_DIR/$NAME.$num.pid
+  echo $"Reloading ${NAME}:${num}: "
+  $DAEMON -d reload --log-file $LOG_FILE --pid-file $pidfile
+  RETVAL=$?
+  [ $RETVAL -eq 0 ] && success || failure
+  echo
+  return $RETVAL
+}
+
+function zap() {
+  num=$1
+  pidfile=$PID_DIR/$NAME.$num.pid
+  rm -f $pidfile
+}
+
+function doAll() {
+  func=$1
+  ii=0
+  while [ "$ii" -lt $INSTANCES ] ; do
+    eval "$func $(($ii))"
+    ii=$(($ii + 1))
+  done
+}
+
+case "$1" in
+  start)
+    doAll start
+    ;;
+  stop)
+    doAll stop
+    ;;
+  restart)
+    doAll stop
+    sleep 1
+    doAll start
+    ;;
+  reload)
+    doAll reload
+    ;;
+  zap)
+    doAll zap
+    ;;
+  *)
+    echo "Usage: $0 {start|stop|restart|reload|zap}"
+    exit 1
+esac
+
+exit $RETVAL

data/script/kestrel

@@ -0,0 +1,93 @@
+#!/bin/sh
+#
+# chkconfig: 345 80 20
+# description: kestrel is a light-weight queue written in scala
+#
+
+# Source function library.
+if [ -f /etc/init.d/functions ] ; then
+        . /etc/init.d/functions
+elif [ -f /etc/rc.d/init.d/functions ] ; then
+        . /etc/rc.d/init.d/functions
+else
+        exit 0
+fi  
+
+prog=kestrel
+USER=daemon
+KESTREL_HOME=/opt/kestrel
+VERSION=1.0
+JAR=$KESTREL_HOME/kestrel-$VERSION.jar
+
+if [ ! -r $JAR ]; then
+  echo "FAIL"
+  echo "*** jar missing - not starting"
+  exit 1
+fi
+
+PIDFILE=/var/run/${prog}.pid
+LOCKFILE=/var/lock/subsys/$prog
+
+HEAP_OPTS="-Xmx2048m -Xms1024m -XX:NewSize=256m"
+JAVA_OPTS="-server -verbosegc -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC $HEAP_OPTS"
+
+start() {
+  echo -n $"Starting $prog: "
+  java $JAVA_OPTS -jar $JAR > /var/log/$prog-startup.log 2>&1 & 
+  RETVAL=$?
+  if [ $RETVAL -eq 0 ]; then
+    echo $! > $PIDFILE
+    success $"$prog startup"
+    touch $LOCKFILE
+  else
+    failure $"$prog startup"
+  fi
+  echo
+  return $RETVAL;
+}
+
+stop() {
+  echo -n $"Stopping $prog: "
+  if [ -f $PIDFILE ]; then
+    killproc -p $PIDFILE $prog
+    RETVAL=$?
+    if [ $RETVAL -eq 0 ]; then
+      rm -f $LOCKFILE
+      rm -f $PIDFILE
+    fi;
+  else
+    RETVAL=1
+    failure;
+  fi
+  echo
+  return $RETVAL;
+}
+
+
+case "$1" in
+  start)
+      start
+      ;;
+
+  stop)
+      stop
+      ;;
+
+  restart)
+      stop
+      start
+      ;;
+
+  condrestart)
+      if [ -f $LOCKFILE ]; then
+          stop
+          start
+      fi
+      ;;
+
+  *)
+      echo $"Usage: $0 {start|stop|restart|condrestart}"
+      exit 1
+esac
+
+exit $RETVAL

data/script/kestrel.sh

@@ -0,0 +1,93 @@
+#!/bin/sh
+#
+# chkconfig: 345 80 20
+# description: kestrel is a light-weight queue written in scala
+#
+
+# Source function library.
+if [ -f /etc/init.d/functions ] ; then
+        . /etc/init.d/functions
+elif [ -f /etc/rc.d/init.d/functions ] ; then
+        . /etc/rc.d/init.d/functions
+else
+        exit 0
+fi  
+
+prog=kestrel
+USER=daemon
+KESTREL_HOME=/opt/kestrel
+VERSION=1.0
+JAR=$KESTREL_HOME/kestrel-$VERSION.jar
+
+if [ ! -r $JAR ]; then
+  echo "FAIL"
+  echo "*** jar missing - not starting"
+  exit 1
+fi
+
+PIDFILE=/var/run/${prog}.pid
+LOCKFILE=/var/lock/subsys/$prog
+
+HEAP_OPTS="-Xmx2048m -Xms1024m -XX:NewSize=256m"
+JAVA_OPTS="-server -verbosegc -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC $HEAP_OPTS"
+
+start() {
+  echo -n $"Starting $prog: "
+  java $JAVA_OPTS -jar $JAR > /var/log/$prog-startup.log 2>&1 & 
+  RETVAL=$?
+  if [ $RETVAL -eq 0 ]; then
+    echo $! > $PIDFILE
+    success $"$prog startup"
+    touch $LOCKFILE
+  else
+    failure $"$prog startup"
+  fi
+  echo
+  return $RETVAL;
+}
+
+stop() {
+  echo -n $"Stopping $prog: "
+  if [ -f $PIDFILE ]; then
+    killproc -p $PIDFILE $prog
+    RETVAL=$?
+    if [ $RETVAL -eq 0 ]; then
+      rm -f $LOCKFILE
+      rm -f $PIDFILE
+    fi;
+  else
+    RETVAL=1
+    failure;
+  fi
+  echo
+  return $RETVAL;
+}
+
+
+case "$1" in
+  start)
+      start
+      ;;
+
+  stop)
+      stop
+      ;;
+
+  restart)
+      stop
+      start
+      ;;
+
+  condrestart)
+      if [ -f $LOCKFILE ]; then
+          stop
+          start
+      fi
+      ;;
+
+  *)
+      echo $"Usage: $0 {start|stop|restart|condrestart}"
+      exit 1
+esac
+
+exit $RETVAL