job_boss 0.2

data/lib/job_boss/boss.rb

@@ -0,0 +1,153 @@
+module JobBoss
+  class Boss
+    class << self
+      # Used to set Boss configuration
+      # Usage:
+      #   Boss.config.sleep_interval = 2
+      def config
+        require 'job_boss/config'
+        @@config ||= Config.new
+      end
+
+      # Used to queue jobs
+      # Usage:
+      #   Boss.queue.math.is_prime?(42)
+      def queue
+        require 'job_boss/queuer'
+        @@queuer ||= Queuer.new
+      end
+    end
+
+    def initialize(options = {})
+      @@config.working_dir          ||= options[:working_dir]
+      @@config.sleep_interval       ||= options[:sleep_interval]
+      @@config.employee_limit       ||= options[:employee_limit]
+      @@config.database_yaml_path   ||= options[:database_yaml_path]
+      @@config.jobs_path            ||= options[:jobs_path]
+
+      @running_jobs = []
+    end
+
+    # Start the boss
+    def start
+      require 'active_record'
+      require 'yaml'
+
+      establish_active_record_connection
+
+      require_job_classes
+
+      require 'job_boss/job'
+
+      migrate
+
+      Signal.trap("HUP") do
+        stop
+      end
+
+      at_exit do
+        stop if Process.pid == BOSS_PID
+      end
+
+      puts "Job Boss started"
+
+      while true
+        unless (children_count = available_employees) > 0 && Job.pending.count > 0
+          sleep(@@config.sleep_interval)
+          next
+        end
+
+        Job.pending_paths.each do |path|
+          job = Job.pending.find_by_path(path)
+          next if job.nil?
+
+          job.dispatch
+          @running_jobs << job
+
+          children_count -= 1
+          break unless children_count > 0
+        end
+
+      end
+    end
+
+    def stop
+      puts "Stopping #{@running_jobs.size} running employees..."
+
+      shutdown_running_jobs
+
+      puts "Job Boss stopped"
+    end
+
+private
+    # Cleans up @running_jobs variable, getting rid of jobs which have
+    # completed, which have been cancelled, or which went MIA
+    def cleanup_running_jobs
+      Job.uncached do
+        @running_jobs = Job.running.where('id in (?)', @running_jobs)
+
+        cancelled_jobs = @running_jobs.select(&:cancelled?)
+        cancelled_jobs.each {|job| kill_job(job) }
+        @running_jobs -= cancelled_jobs
+
+        # Clean out any jobs whos processes have stopped running for some reason
+        @running_jobs = @running_jobs.select do |job|
+          begin
+            Process.kill(0, job.employee_pid.to_i)
+          rescue Errno::ESRCH
+            nil
+          end
+        end
+      end
+    end
+
+    # Total number of employees which can be run
+    def available_employees
+      cleanup_running_jobs
+
+      @@config.employee_limit - @running_jobs.size
+    end
+
+    def establish_active_record_connection
+      @@config.database_yaml_path = File.join(@@config.working_dir, @@config.database_yaml_path) unless @@config.database_yaml_path[0] == ?/
+
+      raise "Database YAML file missing (#{@@config.database_yaml_path})" unless File.exist?(@@config.database_yaml_path)
+
+      config = YAML.load(File.read(@@config.database_yaml_path))
+
+      ActiveRecord::Base.establish_connection(config[@@config.environment])
+    end
+
+    def require_job_classes
+      @@config.jobs_path = File.join(@@config.working_dir, @@config.jobs_path) unless @@config.jobs_path[0] == ?/
+
+      raise "Jobs path missing (#{@@config.jobs_path})" unless File.exist?(@@config.jobs_path)
+
+      Dir.glob(File.join(@@config.jobs_path, '*.rb')).each {|job_class| require job_class }
+    end
+
+    def migrate
+      unless Job.table_exists?
+        require 'migrate'
+        CreateJobs.up
+      end
+    end
+
+    def kill_job(job)
+      begin
+        Process.kill("HUP", job.employee_pid.to_i)
+      rescue Errno::ESRCH
+        nil
+      end
+    end
+
+    def shutdown_running_jobs
+      cleanup_running_jobs
+
+      @running_jobs.each do |job|
+        kill_job(job)
+        job.mark_for_redo
+      end
+    end
+  end
+end

data/lib/job_boss/capistrano.rb

@@ -0,0 +1,8 @@
+# Capistrano task for job_boss.
+#
+# Just add "require 'job_boss/capistrano'" in your Capistrano deploy.rb, and
+# job_boss will be activated after each new deployment.
+
+Capistrano::Configuration.instance(:must_exist).load do
+  after "deploy:update_code", "job_boss:restart"
+end

data/lib/job_boss/configuror.rb

@@ -0,0 +1,40 @@
+module JobBoss
+  class Config
+    attr_accessor :working_dir, :database_yaml_path, :jobs_path, :sleep_interval, :employee_limit, :environment
+
+    def parse_args(argv, options = {})
+      @working_dir          = options[:working_dir] || Dir.pwd
+      @database_yaml_path   = 'config/database.yml'
+      @jobs_path            = 'app/jobs'
+      @sleep_interval       = 0.5
+      @employee_limit       = 4
+      @environment          = 'development'
+
+      require 'optparse'
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: job_boss [start|stop|restart|status|watch] [-- <options>]"
+
+        opts.on("-d", "--database-yaml PATH", "Path for database YAML (defaults to ./config/database.yml)") do |path|
+          @database_yaml_path = path
+        end
+
+        opts.on("-j", "--jobs-path PATH", "Path to folder with job classes (defaults to ./app/jobs)") do |path|
+          @database_yaml_path = path
+        end
+
+        opts.on("-e", "--environment ENV", "Rails environment to use in database YAML file (defaults to 'development')") do |env|
+          @environment = env
+        end
+
+        opts.on("-s", "--sleep-interval INTERVAL", Integer, "Number of seconds for the boss to sleep between checks of the queue (default 0.5)") do |interval|
+          @sleep_interval = interval
+        end
+
+        opts.on("-c", "--child-limit LIMIT", Integer, "Maximum number of employees (default 4)") do |limit|
+          @employee_limit = limit
+        end
+      end.parse!(argv)
+    end
+  end
+end

data/lib/job_boss/job.rb

@@ -0,0 +1,155 @@
+
+module JobBoss
+  class Job < ActiveRecord::Base
+    default_scope order('created_at')
+
+    serialize :args
+    serialize :result
+    serialize :error_backtrace
+
+    scope :pending, where('started_at IS NULL')
+    scope :running, where('started_at IS NOT NULL AND completed_at IS NULL')
+    scope :completed, where('completed_at IS NOT NULL')
+
+    # Method used by the boss to dispatch an employee
+    def dispatch
+      mark_as_started
+      puts "Dispatching Job ##{self.id}"
+
+      pid = fork do
+        $0 = "job_boss - employee (job ##{self.id})"
+        Process.setpriority(Process::PRIO_PROCESS, 0, 19)
+
+        begin
+          mark_employee
+
+          Signal.trap("HUP") do
+            mark_for_redo
+          end
+
+          result = self.class.call_path(self.path, *self.args)
+          self.update_attribute(:result, result)
+        rescue Exception => exception
+          mark_exception(exception)
+          puts "Error running job ##{self.id}!"
+        ensure
+          until mark_as_completed
+            sleep(1)
+          end
+
+          puts "Job ##{self.id} completed, exiting..."
+          Kernel.exit
+        end
+      end
+
+      Process.detach(pid)
+    end
+
+    # Clear out the job and put it back onto the queue for processing
+    def mark_for_redo
+      self.reload
+      self.started_at = nil
+      self.result = nil
+      self.completed_at = nil
+      self.status = nil
+      self.error_message = nil
+      self.error_backtrace = nil
+      self.employee_host = nil
+      self.employee_pid = nil
+      self.save
+    end
+
+    # Is the job complete?
+    def completed?
+      !!completed_at
+    end
+
+    # Mark the job as cancelled so that the boss won't run the job and so that
+    # the employee running the job gets stopped (if it's been dispatched)
+    def cancel
+      mark_as_cancelled
+    end
+
+    # Has the job been cancelled?
+    def cancelled?
+      !!cancelled_at
+    end
+
+    # Did the job succeed?
+    def succeeded?
+      completed_at && (status == 'success')
+    end
+
+    # How long did the job take?
+    def time_taken
+      completed_at - started_at if completed_at && started_at
+    end
+
+    class << self
+      def wait_for_jobs(jobs, sleep_interval = 0.5)
+        running_jobs = jobs.dup
+
+        until Job.running.find_all_by_id(running_jobs.collect(&:id)).empty?
+          sleep(sleep_interval)
+        end
+
+        true
+      end
+
+      def result_hash(jobs)
+        jobs.inject({}) do |hash, job|
+          hash.merge(job.args => job.result)
+        end
+      end
+    end
+
+private
+
+    def mark_as_started
+      update_attributes(:started_at => Time.now)
+    end
+
+    def mark_as_cancelled
+      update_attributes(:cancelled_at => Time.now)
+    end
+
+    def mark_employee
+      require 'socket'
+      update_attributes(:employee_host    => Socket.gethostname,
+                        :employee_pid     => Process.pid)
+    end
+
+    def mark_exception(exception)
+      update_attributes(:status => 'error', :error_message => exception.message, :error_backtrace => exception.backtrace)
+    end
+
+    def mark_as_completed
+      self.status ||= 'success'
+      self.completed_at = Time.now
+      self.save
+    end
+
+    class << self
+      def call_path(path, *args)
+        require 'active_support'
+    
+        raise ArgumentError, "Invalid path (must have #)" unless path.match(/.#./)
+        controller, action = path.split('#')
+    
+        controller_object = begin
+          Kernel.const_get("#{controller.classify}Jobs").new
+        rescue NameError
+          raise ArgumentError, "Invalid controller"
+        end
+    
+        raise ArgumentError, "Invalid path action" unless controller_object.respond_to?(action)
+    
+        controller_object.send(action, *args)
+      end
+    
+      def pending_paths
+        self.pending.except(:order).select('DISTINCT path').collect(&:path)
+      end
+    end
+  end
+end

data/lib/job_boss/queuer.rb

@@ -0,0 +1,37 @@
+module JobBoss
+  class Queuer
+    def method_missing(method_id, *args)
+      require 'active_support'
+
+      method_name = method_id.id2name
+
+      if @class && @controller
+        # In here, we've already figured out the class, so assume the method_missing call is to the method
+
+        if @class.respond_to?(method_name)
+          require 'job_boss/job'
+          path = "#{@controller}##{method_name}"
+
+          @class = nil
+          @controller = nil
+
+          Job.create(:path => path,
+                     :args => args)
+        else
+          raise ArgumentError, "Invalid action"
+        end
+      else
+        # Check to see if there's a jobs class
+        begin
+          @class = Kernel.const_get("#{method_name.classify}Jobs").new
+
+          @controller = method_name
+        rescue NameError
+          raise ArgumentError, "Invalid controller"
+        end
+
+        self
+      end
+    end
+  end
+end

data/lib/migrate.rb

@@ -0,0 +1,27 @@
+class CreateJobs < ActiveRecord::Migration
+  def self.up
+    create_table :jobs do |t|
+      t.string :path
+      t.text :args
+      t.text :result
+      t.datetime :started_at
+      t.datetime :cancelled_at
+      t.datetime :completed_at
+      t.string :status
+      t.string :error_message
+      t.text :error_backtrace
+
+      t.string :employee_host
+      t.string :employee_pid
+
+      t.timestamps
+    end
+
+    add_index :jobs, :path
+    add_index :jobs, :status
+  end
+
+  def self.down
+    drop_table :jobs
+  end
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification 
+name: job_boss
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  version: "0.2"
+platform: ruby
+authors: 
+- Brian Underwood
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-26 00:00:00 -05:00
+default_executable: job_boss
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: daemons
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: job_boss allows you to queue jobs which are unqueued by a "Job Boss" daemon and handed off to workers to process
+email: 
+- ml+job_boss@semi-sentient.com
+executables: 
+- job_boss
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.markdown
+- Rakefile
+- bin/job_boss
+- doc/ActiveRecord.html
+- doc/ActiveRecord/Base.html
+- doc/CreateJobs.html
+- doc/JobBoss.html
+- doc/JobBoss/Boss.html
+- doc/JobBoss/Config.html
+- doc/JobBoss/Job.html
+- doc/JobBoss/Queuer.html
+- doc/Mongrel.html
+- doc/Mongrel/HttpServer.html
+- doc/Passenger.html
+- doc/Passenger/Railz.html
+- doc/Passenger/Railz/RequestHandler.html
+- doc/PhusionPassenger.html
+- doc/PhusionPassenger/Rack.html
+- doc/PhusionPassenger/Rack/RequestHandler.html
+- doc/PhusionPassenger/Railz.html
+- doc/PhusionPassenger/Railz/RequestHandler.html
+- doc/Rakefile.html
+- doc/Spawn.html
+- doc/Spawn/SpawnId.html
+- doc/bin/job_boss.html
+- doc/created.rid
+- doc/index.html
+- doc/lib/job_boss/boss_rb.html
+- doc/lib/job_boss/capistrano_rb.html
+- doc/lib/job_boss/configuror_rb.html
+- doc/lib/job_boss/job_rb.html
+- doc/lib/job_boss/queuer_rb.html
+- doc/lib/migrate_rb.html
+- doc/rdoc.css
+- doc/vendor/spawn/CHANGELOG.html
+- doc/vendor/spawn/LICENSE.html
+- doc/vendor/spawn/init_rb.html
+- doc/vendor/spawn/lib/patches_rb.html
+- doc/vendor/spawn/lib/spawn_rb.html
+- job_boss.gemspec
+- lib/job_boss/boss.rb
+- lib/job_boss/capistrano.rb
+- lib/job_boss/configuror.rb
+- lib/job_boss/job.rb
+- lib/job_boss/queuer.rb
+- lib/migrate.rb
+has_rdoc: true
+homepage: http://github.com/cheerfulstoic/job_boss
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- vendor
+- vendor/spawn/lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Asyncronous, parallel job processing
+test_files: []
+