quebert 0.0.0 → 0.0.1

data/Gemfile

@@ -1,7 +1,6 @@
 source :rubygems
 
 gem 'json'
-gem 'daemons'
 gem 'optitron'
 gem 'beanstalk-client'
 

data/README.rdoc

@@ -7,11 +7,11 @@ A worker queue framework designed around Beanstalk. Features include:
 = Features
 
 * [x] Multiple back-ends (InProcess, Sync, and Beanstalk)
-* [x] Rails/ActiveRecord integration similar to async_observer
+* [ ] Rails/ActiveRecord integration similar to async_observer
 * [ ] Pluggable exception handling (for Hoptoad integration)
 * [ ] Reliable daemonization with pidfile support
 
-[x] = Completed, [ ] = Not Started
+  [x] = Completed, [ ] = Not Started
 
 = How to use
 

data/Rakefile

@@ -11,7 +11,7 @@ begin
     gem.authors = ["Brad Gessler"]
     gem.add_development_dependency "rspec", ">= 1.2.9"
     gem.add_dependency "json"
-    gem.add_dependency "daemons"
+    gem.add_dependency "optitron"
     gem.add_dependency "beanstalk-client"
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end

data/VERSION

@@ -1 +1 @@
-0.0.0
+0.0.1

data/bin/quebert

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+begin
+  require "quebert"
+rescue LoadError
+  quebert_path = File.expand_path('../../lib/', __FILE__)
+  $:.unshift(quebert_path)
+  require 'rubygems'
+  require 'bundler/setup'
+  require 'quebert'
+end
+
+Quebert::CommandLineRunner.dispatch

data/lib/quebert.rb

@@ -1,14 +1,12 @@
-$:.unshift File.join(File.dirname(__FILE__), 'quebert')
-
 module Quebert
-  autoload :Configuration,  'quebert/configuration'
-  autoload :Job,            'quebert/job'
-  autoload :Consumer,       'quebert/consumer'
-  autoload :Backend,        'quebert/backend'
-  autoload :Support,        'quebert/support'
-  autoload :Worker,         'quebert/worker'
-  autoload :Daemonizable,   'quebert/daemonizing'
-  autoload :AsyncSender,    'quebert/async_sender'
+  autoload :Configuration,      'quebert/configuration'
+  autoload :Job,                'quebert/job'
+  autoload :Consumer,           'quebert/consumer'
+  autoload :Backend,            'quebert/backend'
+  autoload :Support,            'quebert/support'
+  autoload :Worker,             'quebert/worker'
+  autoload :CommandLineRunner,  'quebert/command_line_runner'
+  autoload :AsyncSender,        'quebert/async_sender'
   
   class << self
     def configuration
@@ -18,12 +16,17 @@ module Quebert
 
     # Registry for quebert backends
     def backends
-      @backends ||= {}
+      @backends ||= Support::Registry.new
+    end
+    
+    # Make this easier for elsewhere in the app
+    def logger
+      config.logger
     end
   end
   
   # Register built-in Quebert backends
-  Backend.register :beanstalk,  Backend::Beanstalk
-  Backend.register :in_process, Backend::InProcess
-  Backend.register :sync,       Backend::Sync
+  Quebert.backends.register :beanstalk,  Backend::Beanstalk
+  Quebert.backends.register :in_process, Backend::InProcess
+  Quebert.backends.register :sync,       Backend::Sync
 end

data/lib/quebert/backend.rb

@@ -3,9 +3,5 @@ module Quebert
     autoload :InProcess,  'quebert/backend/in_process.rb'
     autoload :Beanstalk,  'quebert/backend/beanstalk.rb'
     autoload :Sync,       'quebert/backend/sync.rb'
-    
-    def self.register(name, backend)
-      Quebert.backends[name.to_sym] = backend
-    end
   end
 end

data/lib/quebert/command_line_runner.rb

@@ -0,0 +1,40 @@
+require 'optitron'
+
+module Quebert
+  class CommandLineRunner < Optitron::CLI
+    
+    desc "Starts a quebert worker"
+    opt "pid-file",     :type => :string
+    opt "log-file",     :type => :string
+    opt "config-file",  :type => :string
+    opt "chdir",        :type => :string
+    def worker
+      if dir = params['chdir']
+        Dir.chdir dir
+      end
+      
+      if pid_file = params['pid-file']
+        Support::PidFile.new(pid_file).write!
+      end
+      
+      if log_path = params['log-file']
+        Quebert.config.log_file_path = log_path
+      end
+      
+      if config = params['config-file'] || auto_config
+        require config
+      end
+      
+      Worker.new.start
+    end
+    
+  private
+    def auto_config
+      rails_env_path = './config/environment.rb'
+      if File.exists?(rails_env_path)
+        Quebert.logger.info "Detected Rails! Setting config-file=#{File.expand_path(rails_env_path)}"
+        rails_env_path
+      end
+    end
+  end
+end

data/lib/quebert/configuration.rb

@@ -1,16 +1,29 @@
+require 'logger'
+
 module Quebert
   class Configuration
-    attr_accessor :backend
+    attr_accessor :backend, :logger
     
-    def self.from_hash(hash)
-      hash, config = Support.symbolize_keys(hash), new
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+    
+    def log_file_path=(path)
+      self.logger = Logger.new(path)
+    end
+    
+    def from_hash(hash)
+      hash = Support.symbolize_keys(hash)
       # Find out backend from the registry and configure
       if backend = Quebert.backends[hash.delete(:backend).to_sym]
         # If the backend supports configuration, do it!
-        p backend
-        config.backend = backend.respond_to?(:configure) ? backend.configure(Support.symbolize_keys(hash)) : backend.new
+        self.backend = backend.respond_to?(:configure) ? backend.configure(Support.symbolize_keys(hash)) : backend.new
       end
-      config
+      self
+    end
+    
+    def self.from_hash(hash)
+      new.from_hash(hash) # Config this puppy up from a config hash
     end
   end
 end

data/lib/quebert/support.rb

@@ -1,5 +1,9 @@
 module Quebert
   module Support
+    
+    autoload :PidFile,  'quebert/support/pid_file'
+    autoload :Registry, 'quebert/support/registry'
+    
     # Borrowed from Rails ActiveSupport
     def self.constantize(camel_cased_word) #:nodoc:
       names = camel_cased_word.split('::')

data/lib/quebert/support/pid_file.rb

@@ -0,0 +1,72 @@
+module Quebert
+  module Support
+    
+    # Deal with all of our pid file stuff
+    class PidFile
+      
+      attr_reader :path
+      
+      ProcessRunning = Class.new(RuntimeError)
+      
+      def initialize(path)
+        @path = path
+      end
+      
+      def write!
+        remove_stale and write
+      end
+      
+      # Read pids and turn them into ints
+      def self.read(file)
+        if File.file?(file) && pid = File.read(file)
+          pid.to_i
+        else
+          nil
+        end
+      end
+      
+      # Tells us if a current process is running
+      def self.running?(pid)
+        Process.getpgid(pid) != -1
+      rescue Errno::EPERM
+        true
+      rescue Errno::ESRCH
+        false
+      end
+      
+    private
+      # If PID file is stale, remove it.
+      def remove_stale
+        if exists? && running?
+          raise ProcessRunning, "#{path} already exists, seems like it's already running (process ID: #{pid}). " +
+                              "Stop the process or delete #{path}."
+        else
+          remove
+        end
+      end
+      
+      def remove
+        File.delete(path) if exists?
+        true
+      end
+      
+      def write
+        File.open(path,"w") { |f| f.write(Process.pid) }
+        File.chmod(0644, path)
+      end
+      
+      def pid
+        self.class.read(path)
+      end
+      
+      def running?
+        self.class.running?(pid)
+      end
+      
+      def exists?
+        File.exist?(path)
+      end
+    end
+    
+  end
+end

data/lib/quebert/support/registry.rb

@@ -0,0 +1,13 @@
+module Quebert
+  module Support
+    class Registry < Hash
+      def register(key, val)
+        self[key.to_sym] = val
+      end
+    
+      def unregister(key)
+        self.delete(key.to_sym)
+      end
+    end
+  end
+end

data/lib/quebert/worker.rb

@@ -2,9 +2,7 @@ require 'logger'
 
 module Quebert
   class Worker
-    attr_accessor :exception_handler, :log_file, :backend
-    
-    include Quebert::Daemonizable
+    attr_accessor :exception_handler, :logger, :backend
     
     def initialize
       yield self if block_given?
@@ -12,18 +10,30 @@ module Quebert
     
     # Start the worker backend and intercept exceptions if a handler is provided
     def start
-      while job = backend.reserve do
+      logger.info "Worker pid##{Process.pid} started with #{backend.class.name} backend"
+      while consumer = backend.reserve do
         begin
-          job.perform
+          log consumer.job, "performing with args #{consumer.job.args.inspect}"
+          consumer.perform
+          log consumer.job, "complete"
         rescue Exception => e
+          log consumer.job, "fault #{e}", :error
           exception_handler ? exception_handler.call(e) : raise(e)
         end
       end
     end
     
   protected
-    def log(message)
-      puts message
+    def log(job, message, level=:info)
+      logger.send(level, "#{job.class.name}##{job.object_id}: #{message}")
+    end
+    
+    def logger
+      @logger ||= Quebert.logger
+    end
+    
+    def backend
+      @backend ||= Quebert.config.backend
     end
   end
 end

data/quebert.gemspec

@@ -5,13 +5,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{quebert}
-  s.version = "0.0.0"
+  s.version = "0.0.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brad Gessler"]
-  s.date = %q{2010-10-03}
+  s.date = %q{2010-10-04}
+  s.default_executable = %q{quebert}
   s.description = %q{A worker queue framework built around beanstalkd}
   s.email = %q{brad@bradgessler.com}
+  s.executables = ["quebert"]
   s.extra_rdoc_files = [
     "LICENSE",
      "README.rdoc"
@@ -24,23 +26,27 @@ Gem::Specification.new do |s|
      "README.rdoc",
      "Rakefile",
      "VERSION",
+     "bin/quebert",
      "lib/quebert.rb",
      "lib/quebert/async_sender.rb",
      "lib/quebert/backend.rb",
      "lib/quebert/backend/beanstalk.rb",
      "lib/quebert/backend/in_process.rb",
      "lib/quebert/backend/sync.rb",
+     "lib/quebert/command_line_runner.rb",
      "lib/quebert/configuration.rb",
      "lib/quebert/consumer.rb",
      "lib/quebert/consumer/base.rb",
      "lib/quebert/consumer/beanstalk.rb",
-     "lib/quebert/daemonizing.rb",
      "lib/quebert/job.rb",
      "lib/quebert/support.rb",
+     "lib/quebert/support/pid_file.rb",
+     "lib/quebert/support/registry.rb",
      "lib/quebert/worker.rb",
      "quebert.gemspec",
      "spec/async_sender_spec.rb",
      "spec/backend_spec.rb",
+     "spec/command_line_runner_spec.rb",
      "spec/configuration_spec.rb",
      "spec/consumer_spec.rb",
      "spec/job_spec.rb",
@@ -58,6 +64,7 @@ Gem::Specification.new do |s|
   s.test_files = [
     "spec/async_sender_spec.rb",
      "spec/backend_spec.rb",
+     "spec/command_line_runner_spec.rb",
      "spec/configuration_spec.rb",
      "spec/consumer_spec.rb",
      "spec/job_spec.rb",
@@ -74,18 +81,18 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_runtime_dependency(%q<json>, [">= 0"])
-      s.add_runtime_dependency(%q<daemons>, [">= 0"])
+      s.add_runtime_dependency(%q<optitron>, [">= 0"])
       s.add_runtime_dependency(%q<beanstalk-client>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<json>, [">= 0"])
-      s.add_dependency(%q<daemons>, [">= 0"])
+      s.add_dependency(%q<optitron>, [">= 0"])
       s.add_dependency(%q<beanstalk-client>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
     s.add_dependency(%q<json>, [">= 0"])
-    s.add_dependency(%q<daemons>, [">= 0"])
+    s.add_dependency(%q<optitron>, [">= 0"])
     s.add_dependency(%q<beanstalk-client>, [">= 0"])
   end
 end

data/spec/backend_spec.rb

@@ -6,9 +6,14 @@ describe Backend do
   end
   
   it "should register backends" do
-    Quebert::Backend.register :twenty, 20
+    Quebert.backends.register :twenty, 20
     Quebert.backends[:twenty].should eql(20)
   end
+  
+  it "should unregister backends" do
+    Quebert.backends.unregister :twenty
+    Quebert.backends[:twenty].should be_nil
+  end
 end
 
 describe Backend::InProcess do

data/spec/command_line_runner_spec.rb

@@ -0,0 +1,77 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe CommandLineRunner do
+  before(:all) do
+    Quebert.config.backend = Backend::InProcess.new
+  end
+  
+  context "log-file" do
+    it "should write log file" do
+      clean_file 'log.log' do
+        lambda{
+          CommandLineRunner.dispatch(%w(worker --log-file=log.log))
+        }.should change { File.read('log.log') if File.exists?('log.log') }
+      end
+    end
+  end
+  
+  context "pid-file" do
+    it "should write pid" do
+      clean_file 'pid.pid' do
+        File.exists?('pid').should be_false
+        CommandLineRunner.dispatch(%w(worker --pid-file=pid.pid))
+        Support::PidFile.read('pid.pid').should eql(Process.pid)
+      end
+    end
+    
+    it "should remove stale" do
+      clean_file 'pid.pid', "-1" do
+        CommandLineRunner.dispatch(%w(worker --pid-file=pid.pid))
+        Support::PidFile.read('pid.pid').should eql(Process.pid)
+      end
+    end
+    
+    it "should complain if the pid is already running" do
+      clean_file 'pid.pid', Process.pid do
+        lambda{
+          CommandLineRunner.dispatch(%w(worker --pid-file=pid.pid))
+        }.should raise_exception(Support::PidFile::ProcessRunning)
+        Support::PidFile.read('pid.pid').should eql(Process.pid)
+      end
+    end
+  end
+  
+  context "config-file" do
+    it "should auto-detect rails environment file" do
+      clean_file './config/environment.rb', "raise 'RailsConfig'" do
+        lambda{
+          CommandLineRunner.dispatch(%w(worker))
+        }.should raise_exception('RailsConfig')
+      end
+    end
+    
+    it "should run config file" do
+      clean_file './super_awesome.rb', "raise 'SuperAwesome'" do
+        lambda{
+          CommandLineRunner.dispatch(%w(worker --config-file=super_awesome.rb))
+        }.should raise_exception('SuperAwesome')
+      end
+    end
+    
+  end
+  
+  context "chdir" do
+    before(:each) do
+      @chdir = Dir.pwd
+    end
+    
+    it "should change chdir" do
+      CommandLineRunner.dispatch(%w(worker --chdir=/))
+      Dir.pwd.should eql('/')
+    end
+    
+    after(:each) do
+      Dir.chdir(@chdir)
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 describe Configuration do
   context "from hash" do
     before(:all) do
-      @config = Configuration.from_hash('backend' => 'beanstalk', 'host' => 'localhost:11300', 'tube' => 'quebert-config-test')
+      @config = Configuration.new.from_hash('backend' => 'beanstalk', 'host' => 'localhost:11300', 'tube' => 'quebert-config-test')
     end
     
     it "should configure backend" do

data/spec/spec_helper.rb

@@ -7,10 +7,23 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'quebert'
 require 'spec'
 require 'spec/autorun'
+require 'logger'
 
 Spec::Runner.configure do |config|
 end
 
 include Quebert
+Quebert.config.logger = Logger.new('/dev/null') # Shhh...
 
-require 'jobs'
+require 'jobs'
+
+def clean_file(path, contents=nil, &block)
+  FileUtils.remove_entry(path) if File.exists?(path)
+  FileUtils.mkdir_p(File.dirname(path))
+  begin
+    File.open(path, 'w'){|f| f.write(contents == :empty ? nil : contents) } unless contents.nil?
+    block.call
+  ensure
+    FileUtils.remove_entry(path) if File.exists?(path) and path != './'  # Yeah! This has happened before :(
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: quebert
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: false
   segments: 
   - 0
   - 0
-  - 0
-  version: 0.0.0
+  - 1
+  version: 0.0.1
 platform: ruby
 authors: 
 - Brad Gessler
@@ -15,8 +15,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-03 00:00:00 -07:00
-default_executable: 
+date: 2010-10-04 00:00:00 -07:00
+default_executable: quebert
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -49,7 +49,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: daemons
+  name: optitron
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -78,8 +78,8 @@ dependencies:
   version_requirements: *id004
 description: A worker queue framework built around beanstalkd
 email: brad@bradgessler.com
-executables: []
-
+executables: 
+- quebert
 extensions: []
 
 extra_rdoc_files: 
@@ -93,23 +93,27 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
+- bin/quebert
 - lib/quebert.rb
 - lib/quebert/async_sender.rb
 - lib/quebert/backend.rb
 - lib/quebert/backend/beanstalk.rb
 - lib/quebert/backend/in_process.rb
 - lib/quebert/backend/sync.rb
+- lib/quebert/command_line_runner.rb
 - lib/quebert/configuration.rb
 - lib/quebert/consumer.rb
 - lib/quebert/consumer/base.rb
 - lib/quebert/consumer/beanstalk.rb
-- lib/quebert/daemonizing.rb
 - lib/quebert/job.rb
 - lib/quebert/support.rb
+- lib/quebert/support/pid_file.rb
+- lib/quebert/support/registry.rb
 - lib/quebert/worker.rb
 - quebert.gemspec
 - spec/async_sender_spec.rb
 - spec/backend_spec.rb
+- spec/command_line_runner_spec.rb
 - spec/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/job_spec.rb
@@ -155,6 +159,7 @@ summary: A worker queue framework built around beanstalkd
 test_files: 
 - spec/async_sender_spec.rb
 - spec/backend_spec.rb
+- spec/command_line_runner_spec.rb
 - spec/configuration_spec.rb
 - spec/consumer_spec.rb
 - spec/job_spec.rb

data/lib/quebert/daemonizing.rb

@@ -1,144 +0,0 @@
-require 'etc'
-require 'daemons'
-require 'fileutils'
-
-module Process
-  # Returns +true+ the process identied by +pid+ is running.
-  def running?(pid)
-    Process.getpgid(pid) != -1
-  rescue Errno::EPERM
-    true
-  rescue Errno::ESRCH
-    false
-  end
-  module_function :running?
-end
-
-module Quebert
-  module Daemonizable
-    attr_accessor :pid_file, :log_file
-    
-    PidFileExist = Class.new(RuntimeError)
-    
-    def self.included(base)
-      base.extend ClassMethods
-    end
-    
-    def daemonize
-      raise ArgumentError, 'You must specify a pid_file to daemonize' unless pid_file
-      
-      remove_stale_pid_file
-      
-      pwd = Dir.pwd # Current directory is changed during daemonization, so store it
-      # HACK we need to create the directory before daemonization to prevent a bug under 1.9
-      #      ignoring all signals when the directory is created after daemonization.
-      FileUtils.mkdir_p File.dirname(pid_file)
-      # Daemonize.daemonize(File.expand_path(@log_file), "quebert worker")
-      Daemonize.daemonize(File.expand_path(@log_file), "quebert")
-      Dir.chdir(pwd)
-      write_pid_file
-    end
-    
-    def pid
-      File.exist?(pid_file) ? open(pid_file).read.to_i : nil
-    end
-    
-    # Register a proc to be called to restart the server.
-    def on_restart(&block)
-      @on_restart = block
-    end
-    
-    # Restart the server.
-    def restart
-      if @on_restart
-        log '>> Restarting ...'
-        stop
-        remove_pid_file
-        @on_restart.call
-        exit!
-      end
-    end
-    
-    module ClassMethods
-      # Send a QUIT or INT (if timeout is +0+) signal the process which
-      # PID is stored in +pid_file+.
-      # If the process is still running after +timeout+, KILL signal is
-      # sent.
-      def kill(pid_file, timeout=60)
-        if timeout == 0
-          send_signal('INT', pid_file, timeout)
-        else
-          send_signal('QUIT', pid_file, timeout)
-        end
-      end
-      
-      # Restart the server by sending HUP signal.
-      def restart(pid_file)
-        send_signal('HUP', pid_file)
-      end
-      
-      # Send a +signal+ to the process which PID is stored in +pid_file+.
-      def send_signal(signal, pid_file, timeout=60)
-        if pid = read_pid_file(pid_file)
-          Logging.log "Sending #{signal} signal to process #{pid} ... "
-          Process.kill(signal, pid)
-          Timeout.timeout(timeout) do
-            sleep 0.1 while Process.running?(pid)
-          end
-        else
-          Logging.log "Can't stop process, no PID found in #{pid_file}"
-        end
-      rescue Timeout::Error
-        Logging.log "Timeout!"
-        force_kill pid_file
-      rescue Interrupt
-        force_kill pid_file
-      rescue Errno::ESRCH # No such process
-        Logging.log "process not found!"
-        force_kill pid_file
-      end
-      
-      def force_kill(pid_file)
-        if pid = read_pid_file(pid_file)
-          Logging.log "Sending KILL signal to process #{pid} ... "
-          Process.kill("KILL", pid)
-          File.delete(pid_file) if File.exist?(pid_file)
-        else
-          Logging.log "Can't stop process, no PID found in #{pid_file}"
-        end
-      end
-      
-      def read_pid_file(file)
-        if File.file?(file) && pid = File.read(file)
-          pid.to_i
-        else
-          nil
-        end
-      end
-    end
-    
-  protected
-    def remove_pid_file
-      File.delete(pid_file) if pid_file && File.exists?(pid_file)
-    end
-    
-    def write_pid_file
-      log ">> Writing PID to #{pid_file}"
-      open(pid_file,"w") { |f| f.write(Process.pid) }
-      File.chmod(0644, pid_file)
-    end
-    
-    # If PID file is stale, remove it.
-    def remove_stale_pid_file
-      if File.exist?(pid_file)
-        if pid && Process.running?(pid)
-          raise PidFileExist, "#{pid_file} already exists, seems like it's already running (process ID: #{pid}). " +
-                              "Stop the process or delete #{pid_file}."
-        else
-          log ">> Deleting stale PID file #{pid_file}"
-          remove_pid_file
-        end
-      end
-    end
-  end
-end