famoseagle-sweat_shop 0.8.2 → 1.0.0

data/Rakefile

@@ -14,7 +14,6 @@ begin
     s.authors = ["Amos Elliston"]
     s.files =  FileList["[A-Z]*", "{lib,test,config}/**/*"]
     s.add_dependency('famoseagle-carrot', '= 0.6.0')
-    s.add_dependency('wonko9-i_can_daemonize', '= 0.7.1')
   end
 rescue LoadError
   puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:major: 0
-:minor: 8
-:patch: 2
+:major: 1
+:minor: 0
+:patch: 0

data/config/defaults.yml

@@ -1,8 +1,9 @@
 # default options
-servers:
-  - localhost:5672
+default: 
+  queue: rabbit
+  host: localhost
+  port: 5672
+  user: 'guest'
+  pass: 'guest'
+  vhost: '/'
 enable: true
-queue: rabbit
-user: 'guest'
-pass: 'guest'
-vhost: '/'

data/config/sweatshop.yml

@@ -1,12 +1,18 @@
 development:
-  servers:
-    - localhost:5672
+  default:
+    queue: rabbit
+    host: localhost 
+    port: 5672
   enable: true
 test:
-  servers:
-    - localhost:5672
-  enable: false
+  default:
+    queue: rabbit
+    host: localhost 
+    port: 5672
+  enable: true
 production:
-  servers:
-    - localhost:5672
+  default:
+    queue: rabbit
+    host: localhost 
+    port: 5672
   enable: true

data/lib/message_queue/rabbit.rb

@@ -3,10 +3,7 @@ module MessageQueue
   class Rabbit < Base
 
     def initialize(opts={})
-      @servers     = opts['servers']
-      @host, @port = @servers.first.split(':')
-      @port        = @port.to_i
-      @opts        = opts
+      @opts = opts
     end
 
     def delete(queue)
@@ -59,8 +56,8 @@ module MessageQueue
 
     def client
       @client ||= Carrot.new(
-        :host  => @host, 
-        :port  => @port, 
+        :host  => @opts['host'], 
+        :port  => @opts['port'].to_i, 
         :user  => @opts['user'], 
         :pass  => @opts['pass'], 
         :vhost => @opts['vhost']

data/lib/sweat_shop.rb

@@ -47,6 +47,9 @@ module SweatShop
           end
         end
         if stop?
+          workers.each do |worker|
+            worker.stop
+          end
           queue.stop
           exit 
         end
@@ -107,16 +110,18 @@ module SweatShop
     puts '-' * (max_width + 10)
   end
 
-  def queue
-    @queue ||= begin 
-      queue = config['queue'] || 'rabbit'
-      queue = constantize("MessageQueue::#{queue.capitalize}")
-      queue.new(config)
+  def queue(type = 'default')
+    @queues ||= {}
+    @queues[type] ||= begin 
+      qconfig = config[type] || config['default']
+      qtype   = qconfig['queue'] || 'rabbit'
+      queue   = constantize("MessageQueue::#{qtype.capitalize}")
+      queue.new(qconfig)
     end
   end
 
-  def queue=(queue)
-    @queue = queue
+  def queue=(queue, type = 'default')
+    @queues[type] = queue
   end
 
   def log(msg)

data/lib/sweat_shop/daemoned.rb

@@ -0,0 +1,405 @@
+require 'optparse'
+require 'timeout'    
+
+module Daemoned
+  class DieTime      < StandardError; end
+  class TimeoutError < StandardError; end
+
+  def self.included(base)
+    base.extend ClassMethods           
+    base.initialize_options
+  end
+
+  class Config
+    METHODS = [:script_path]
+    CONFIG = {}    
+    def method_missing(name, *args)
+      name = name.to_s.upcase.to_sym
+      if name.to_s =~ /^(.*)=$/
+        name = $1.to_sym
+        CONFIG[name] = args.first
+      else
+        CONFIG[name]
+      end
+    end    
+  end
+
+  module ClassMethods  
+    
+    def initialize_options    
+      @@config             = Config.new
+      @@config.script_path = File.expand_path(File.dirname($0))
+      $0                   = script_name
+    end
+    
+    def parse_options
+      opts = OptionParser.new do |opt|
+        opt.banner = "Usage: #{script_name} [options] [start|stop]"
+
+        opt.on_tail('-h', '--help', 'Show this message') do
+          puts opt
+          exit(1)
+        end
+
+        opt.on('--loop-every=SECONDS', 'How long to sleep between each loop') do |value|
+          options[:loop_every] = value 
+        end
+
+        opt.on('-t', '--ontop', 'Stay on top (does not daemonize)') do
+          options[:ontop] = true
+        end
+
+        opt.on('--instances=NUM', 'Allow multiple instances to run simultaneously? 0 for infinite. default: 1') do |value|
+          self.instances = value.to_i
+        end
+
+        opt.on('--log-file=LOGFILE', 'Logfile to log to') do |value|
+          options[:log_file] = File.expand_path(value)
+        end
+
+        opt.on('--pid-file=PIDFILE', 'Location of pidfile') do |value|
+          options[:pid_file] = File.expand_path(value)
+        end
+
+        opt.on('--no-log-prefix', 'Do not prefix PID and date/time in log file.') do
+          options[:log_prefix] = false
+        end
+      end
+      
+      extra_args.each do |arg|
+        opts.on(*arg.first) do |value|
+          arg.last.call(value) if arg.last
+        end
+      end
+
+      opts.parse!
+      
+      if ARGV.include?('stop')                                                         
+        stop
+      elsif ARGV.include?('reload')
+        kill('HUP')
+        exit
+      elsif not ARGV.include?('start') and not ontop?
+        puts opts.help
+      end
+    end    
+
+    def arg(*args, &block)
+      self.extra_args << [args, block]
+    end
+
+    def extra_args
+      @extra_args ||= [] 
+    end
+
+    def callbacks
+      @callbacks ||= {}
+    end
+
+    def options
+      @options ||= {}
+    end
+
+    def options=(options)
+      @options = options
+    end
+
+    def config
+      yield @@config
+    end
+    
+    def before(&block)
+      callbacks[:before] = block
+    end
+
+    def after(&block)
+      callbacks[:after] = block
+    end
+
+    def sig(*signals, &block)
+      signals.each do |s|
+        callbacks["sig_#{s}".to_sym] = block
+      end
+    end
+
+    def die_if(method=nil,&block)
+      options[:die_if] = method || block
+    end
+
+    def exit_if(method=nil,&block)
+      options[:exit_if] = method || block
+    end
+
+    def callback!(callback)
+      callbacks[callback].call if callbacks[callback]
+    end
+
+    # options may include:
+    #
+    # <tt>:loop_every</tt> Fixnum (DEFAULT 0)
+    #  How many seconds to sleep between calls to your block
+    #
+    # <tt>:timeout</tt> Fixnum (DEFAULT 0)
+    #  Timeout in if block does not execute withing passed number of seconds
+    #
+    # <tt>:kill_timeout</tt> Fixnum (DEFAULT 120)
+    #  Wait number of seconds before using kill -9 on daemon
+    #
+    # <tt>:die_on_timeout</tt> BOOL (DEFAULT False)
+    #  Should the daemon continue running if a block times out, or just run the block again
+    #
+    # <tt>:ontop</tt> BOOL (DEFAULT False)
+    #  Do not daemonize.  Run in current process
+    #
+    # <tt>:before</tt> BLOCK
+    #  Run this block after daemonizing but before begining the daemonize loop.
+    #  You can also define the before block by putting a before do/end block in your class.
+    #
+    # <tt>:after</tt> BLOCK
+    #  Run this block before program exists.  
+    #  You can also define the after block by putting an after do/end block in your class.
+    #
+    # <tt>:die_if</tt> BLOCK
+    #  Run this check after each iteration of the loop.   If the block returns true, throw a DieTime exception and exit
+    #  You can also define the after block by putting an die_if do/end block in your class.
+    #      
+    # <tt>:exit_if</tt> BLOCK
+    #  Run this check after each iteration of the loop.   If the block returns true, exit gracefully
+    #  You can also define the after block by putting an exit_if do/end block in your class.
+    #
+    # <tt>:log_prefix</tt> BOOL (DEFAULT true)
+    #  Prefix log file entries with PID and timestamp
+    def daemonize(opts={}, &block)
+      self.options = opts
+      parse_options
+      return unless ok_to_start?
+
+      puts "Starting #{script_name}..."
+      puts "Logging to: #{log_file}" unless ontop?
+      
+      unless ontop?
+        safefork do
+          open(pid_file, 'w'){|f| f << Process.pid }
+          at_exit { remove_pid! }
+    
+          trap('TERM') { callback!(:sig_term)                            }
+          trap('INT')  { callback!(:sig_int)  ; Process.kill('TERM', $$) }
+          trap('HUP')  { callback!(:sig_hup)                             }
+
+          sess_id = Process.setsid
+          reopen_filehandes
+
+          begin
+            at_exit { callback!(:after) }
+            callback!(:before)
+            run_block(&block)
+          rescue SystemExit
+          rescue Exception => e
+            $stdout.puts "Something bad happened #{e.inspect} #{e.backtrace.join("\n")}"
+          end            
+        end
+      else
+        begin
+          callback!(:before)
+          run_block(&block)
+        rescue SystemExit, Interrupt
+          callback!(:after)
+        end
+      end
+    end
+
+  private
+    
+    def run_block(&block)
+      loop do
+        if options[:timeout]
+          begin
+            Timeout::timeout(options[:timeout].to_i) do
+              block.call if block              
+            end
+          rescue Timeout::Error => e
+            if options[:die_on_timeout]
+              raise TimeoutError.new("#{self} timed out after #{options[:timeout]} seconds while executing block in loop")
+            else
+              $stderr.puts "#{self} timed out after #{options[:timeout]} seconds while executing block in loop #{e.backtrace.join("\n")}"
+            end
+          end            
+        else
+          block.call if block
+        end
+
+        if options[:loop_every]
+          sleep options[:loop_every].to_i
+        elsif not block
+          sleep 0.1
+        end
+
+        break if should_exit?
+        raise DieTime.new('Die if conditions were met!') if should_die?
+      end                    
+      exit(0)
+    end
+
+    def should_die?
+      die_if = options[:die_if]
+      if die_if
+        if die_if.is_a?(Symbol) or die_if.is_a?(String)
+          self.send(die_if)
+        elsif die_if.is_a?(Proc)
+          die_if.call
+        end
+      else
+        false
+      end
+    end
+
+    def should_exit?
+      exit_if = options[:exit_if]
+      if exit_if
+        if exit_if.is_a?(Symbol) or exit_if.is_a?(String)
+          self.send(exit_if.to_sym)
+        elsif exit_if.is_a?(Proc)
+          exit_if.call
+        end
+      else
+        false
+      end
+    end
+
+    def ok_to_start?
+      return true if pid.nil?
+
+      if process_alive?
+        $stderr.puts "#{script_name} is already running"
+        return false          
+      else
+        $stderr.puts "Removing stale pid: #{pid}..."
+      end
+
+      true
+    end
+
+    def stop
+      puts "Stopping #{script_name}..."
+      kill
+      exit
+    end     
+
+    def kill(signal = 'TERM')
+      if pid.nil?
+        $stderr.puts "#{script_name} doesn't appear to be running"
+        exit(1)
+      end
+      $stdout.puts("Sending pid #{pid} signal #{signal}...")
+      begin
+        Process.kill(signal, pid)             
+        return if signal == 'HUP' 
+        if pid_running?(options[:kill_timeout] || 120)
+          $stdout.puts("Using kill -9 #{pid}")
+          Process.kill(9, pid)
+        else
+          $stdout.puts("Process #{pid} stopped")
+        end
+      rescue Errno::ESRCH
+       $stdout.puts("Couldn't #{signal} #{pid} as it wasn't running")
+      end
+    end               
+    
+    def pid_running?(time_to_wait = 0)
+      times_to_check = 1
+      if time_to_wait > 0.5
+        times_to_check = (time_to_wait / 0.5).to_i
+      end
+      times_to_check.times do
+        return false unless process_alive?
+        sleep 0.5
+      end
+      true
+    end
+          
+    def safefork(&block)
+      fork_tries ||= 0
+      fork(&block)
+    rescue Errno::EWOULDBLOCK
+      raise if fork_tries >= 20
+      fork_tries += 1
+      sleep 5
+      retry
+    end
+
+    def process_alive?
+      Process.kill(0, pid)
+      true
+    rescue Errno::ESRCH => e
+      false
+    end  
+
+    LOG_FORMAT  = '%-6d %-19s %s'
+    TIME_FORMAT = '%Y/%m/%d %H:%M:%S'
+    def reopen_filehandes
+      STDIN.reopen('/dev/null')
+      STDOUT.reopen(log_file, 'a')
+      STDOUT.sync = true          
+      STDERR.reopen(STDOUT)
+      if log_prefix?
+        def STDOUT.write(string)
+          if @no_prefix
+            @no_prefix = false if string[-1, 1] == "\n"
+          else
+            string = LOG_FORMAT % [$$, Time.now.strftime(TIME_FORMAT), string]
+            @no_prefix = true              
+          end
+          super(string)
+        end
+      end
+    end
+
+    def remove_pid!
+      if File.file?(pid_file) and File.read(pid_file).to_i == $$
+        File.unlink(pid_file)
+      end
+    end
+
+    def ontop?
+      options[:ontop]
+    end
+
+    def log_prefix?
+      options[:log_prefix] || true
+    end                    
+    
+    LOG_PATHS = ['log/', 'logs/', '../log/', '../logs/', '../../log', '../../logs', '.']
+    LOG_PATHS.unshift("#{RAILS_ROOT}/log") if defined?(RAILS_ROOT)
+    def log_dir
+      options[:log_dir] ||= begin
+        LOG_PATHS.detect do |path|
+          File.exists?(File.expand_path(path))        
+        end
+      end               
+    end
+    
+    def log_file
+      options[:log_file] ||= File.expand_path("#{log_dir}/#{script_name}.log")
+    end
+
+    def pid_dir
+      options[:pid_dir] ||= log_dir
+    end
+
+    def pid_file
+      options[:pid_file] ||= File.expand_path("#{pid_dir}/#{script_name}.pid")
+    end
+    
+    def pid
+      @pid ||= File.file?(pid_file) ? File.read(pid_file).to_i : nil
+    end
+
+    def script_name
+      @script_name ||= File.basename($0).gsub('.rb', '')
+    end
+
+    def script_name=(script_name)
+      @script_name = script_name
+    end
+  end
+end

data/lib/sweat_shop/sweatd.rb

@@ -1,12 +1,12 @@
-require File.dirname(__FILE__) + '/../sweat_shop'
-require 'i_can_daemonize'
+require File.dirname(__FILE__) + '/daemoned'
 
 module SweatShop
   class Sweatd
-    include ICanDaemonize
+    include Daemoned
     queues     = []
     groups     = []
     rails_root = nil
+    start_cmd  = "ruby #{__FILE__} start #{ARGV.reject{|a| a == 'start'}.join(' ')}"
 
     arg '--workers=Worker,Worker', 'Workers to service (Default is all)' do |value|
       queues = value.split(',')
@@ -32,12 +32,21 @@ module SweatShop
       puts "Shutting down sweatd..."
       SweatShop.stop
     end
+
+    sig(:hup) do
+      puts "Received HUP"
+      SweatShop.stop
+      remove_pid!
+      puts "Restarting sweatd with #{start_cmd}..."
+      `#{start_cmd}`        
+    end
     
     before do
       if rails_root
         puts "Loading Rails..."
         require rails_root + '/config/environment' 
       end
+      require File.dirname(__FILE__) + '/../sweat_shop'
     end
 
     daemonize(:kill_timeout => 20) do

data/lib/sweat_shop/worker.rb

@@ -95,7 +95,7 @@ module SweatShop
     end
 
     def self.queue
-      SweatShop.queue
+      SweatShop.queue(queue_group.to_s)
     end
 
     def self.workers
@@ -126,6 +126,14 @@ module SweatShop
       end
     end
 
+    def self.stop
+      instance.stop
+    end
+
+    # called before we exit -- subclass can implement this method 
+    def stop; end;
+
+
     def self.queue_group(group=nil)
       group ? meta_def(:_queue_group){ group } : _queue_group
     end

data/test/hello_worker.rb

@@ -1,7 +1,4 @@
-# hack for functional tests
-require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_extended'
-require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_util'
-
+require File.dirname(__FILE__) + '/../lib/sweat_shop'
 class HelloWorker < SweatShop::Worker
   TEST_FILE = File.dirname(__FILE__) + '/test.txt' unless defined?(TEST_FILE)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: famoseagle-sweat_shop
 version: !ruby/object:Gem::Version 
-  version: 0.8.2
+  version: 1.0.0
 platform: ruby
 authors: 
 - Amos Elliston
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-29 00:00:00 -07:00
+date: 2009-07-17 00:00:00 -07:00
 default_executable: sweatd
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,16 +22,6 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.6.0
     version: 
-- !ruby/object:Gem::Dependency 
-  name: wonko9-i_can_daemonize
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        version: 0.7.1
-    version: 
 description: TODO
 email: amos@geni.com
 executables: 
@@ -51,9 +41,9 @@ files:
 - config/sweatshop.yml
 - lib/message_queue/base.rb
 - lib/message_queue/kestrel.rb
-- lib/message_queue/rabbit-async.rb
 - lib/message_queue/rabbit.rb
 - lib/sweat_shop.rb
+- lib/sweat_shop/daemoned.rb
 - lib/sweat_shop/metaid.rb
 - lib/sweat_shop/sweatd.rb
 - lib/sweat_shop/worker.rb

data/lib/message_queue/rabbit-async.rb

@@ -1,83 +0,0 @@
-require 'mq'
-module MessageQueue
-  class Rabbit < Base
-    attr_accessor :em_thread
-
-    def initialize(opts={})
-      @servers = opts['servers']
-      @info = {}
-      @host, @port = @servers.first.split(':')
-      @port = @port.to_i
-    end
-
-    def queue_size(queue)
-      num = 0
-      client.queue(queue).status{|messages, consumers| num = messages}
-      num
-    end
-
-    def enqueue(queue, data)
-      client.queue(queue, :durable => true).publish(Marshal.dump(data), :persistent => true)
-    end
-
-    def dequeue(queue)
-      client.queue(queue).pop do |info, task|
-        @info[queue] = info
-        return Marshal.load(task)
-      end
-    end
-
-    def confirm(queue)
-      if @info[queue] 
-        @info[queue].ack
-        @info[queue] = nil
-      end
-    end
-
-    def client
-      @client ||= begin 
-        start_em
-        if servers
-          MQ.new(AMQP.connect(:host => @host, :port => @port)) 
-        else
-          MQ.new
-        end
-      end
-    end
-
-    def start_em
-      if em_thread.nil? and not EM.reactor_running?
-        self.em_thread = Thread.new{EM.run}
-        ['INT', 'TERM'].each do |sig|
-          old = trap(sig) do 
-            stop
-            old.call
-          end
-        end
-      end
-    end
-
-    def subscribe?
-      true
-    end
-
-    def subscribe(queue, &block)
-      AMQP.start(:host => @host, :port => @port) do
-        mq = MQ.new
-        mq.send(AMQP::Protocol::Basic::Qos.new(:prefetch_size => 0, :prefetch_count => 1, :global => false))
-        mq.queue(queue, :durable => true).subscribe(:ack => true) do |info, task|
-          if task
-            @info[queue] = info
-            task = Marshal.load(task)
-            block.call(task)
-          end
-        end
-      end
-    end
-
-    def stop
-      em_thread.join(0.15) unless em_thread.nil?
-      AMQP.stop{ EM.stop }
-    end
-  end
-end