taf2-rjqueue 0.1.4

data/lib/jobs/base.rb

@@ -0,0 +1,58 @@
+module Jobs
+  class Base
+    attr_reader :record, :logger, :lock
+
+    def initialize(job_record, logger, lock)
+      @record = job_record
+      @logger = logger
+      @lock = lock
+    end
+
+    def execute
+    end
+
+    # really run the execute method
+    def real_execute
+      timer = Time.now
+
+      res = execute
+
+      @record.duration = Time.now - timer
+
+      @record.status = res ? "complete" : "error"
+
+    rescue Object => e
+      # add error details to the record
+      @record.details ||= ""
+      @record.details << "#{e.message}\n#{e.backtrace.join("\n")}"
+
+      # log the error
+      @logger.error "#{e.message}\n#{e.backtrace.join("\n")}"
+      @record.status = "error"
+    ensure
+      # it's no longer processing, so we must have landed here by error
+      @record.status = 'error' if @record.status == 'processing'
+      @record.locked = false
+      @record.save
+    end
+
+    protected
+
+    #
+    # call a process: capture the command output and status code
+    # returns [status, output]
+    def call(cmd)
+      rd, wr = IO.pipe
+      pid = fork do
+        $stdout.reopen wr
+        $stderr.reopen wr
+        $stdin.reopen rd
+        exec cmd
+      end
+      wr.close
+      pid, status = Process.wait2(pid)
+      [status, rd.read]
+    end
+
+  end
+end

data/lib/jobs/client.rb

@@ -0,0 +1,4 @@
+require 'jobs/initializer'
+require 'jobs/scheduler'
+require 'jobs/job'
+require 'jobs/base'

data/lib/jobs/config.rb

@@ -0,0 +1,13 @@
+module Jobs
+  class Config
+
+    def self.host_for_job(name)
+      if defined?(::Jobs::HostMap) and ::Jobs::HostMap.key?(name)
+        ::Jobs::HostMap[name]
+      else
+        {:host => ::Jobs::Keys[:host], :port => ::Jobs::Keys[:port]}
+      end
+    end
+
+  end
+end

data/lib/jobs/initializer.rb

@@ -0,0 +1,39 @@
+require 'yaml'
+require 'active_record'
+
+module Jobs
+  class Initializer
+    def self.run!(job_root_path,job_config_path,env)
+      ::ActiveRecord::Base.send(:include,Jobs::Scheduler)
+      # define the Jobs::Keys constant, e.g. RAILS_ROOT/config/jobs.yml
+      eval %{::Jobs::Keys=HashWithIndifferentAccess.new(YAML.load_file(job_config_path)[env])}
+      # define the jobs root e.g. RAILS_ROOT/apps/jobs
+      eval %{::Jobs::Root=job_root_path}
+    end
+
+    def self.ready?
+      defined?(::Jobs::Root) and defined?(::Jobs::Keys)
+    end
+
+    def self.rails!
+      run! "#{RAILS_ROOT}/app/jobs", "#{RAILS_ROOT}/config/jobs.yml", RAILS_ENV
+    end
+
+    # Configure client to be selective about which job servers to send work
+    #
+    # Jobs::Initializer.config do|cfg|
+    #
+    #  cfg[:sphinx] = [ {:host => '192.168.1.102', :port => 4321} ]
+    #  cfg[:video_encoder] = [ {:host => '192.168.1.102', :port => 4321} ]
+    #
+    def self.config
+      config = {}
+      yield config
+      eval %{::Jobs::HostMap=config}
+    end
+
+    def self.test!
+      run! File.join(File.dirname(__FILE__),'..','..','tests','jobs'), File.join(File.dirname(__FILE__),'..','..','config','jobs.yml'), 'test'
+    end
+  end
+end

data/lib/jobs/job.rb

@@ -0,0 +1,40 @@
+module Jobs
+  class Job < ActiveRecord::Base
+    serialize :data
+    belongs_to :taskable, :polymorphic => true
+
+    #
+    # check if the job is currently being processed
+    #
+    # e.g. status == 'processing' or status == 'pending'
+    #
+    def busy?
+      self.status != 'processing' and self.status != 'pending'
+    end
+
+    def instance(logger_instance, lock)
+      load("#{Jobs::Root}/#{name}_job.rb")
+      klass = "#{name}_job".camelize.constantize
+      klass.new(self,logger_instance, lock)
+    rescue Object => e
+      logger.error "#{e.message}\n#{e.backtrace.join("\n")}"
+      return nil
+    end
+    
+    def retry
+      return false if locked
+      self.status = 'pending'
+      signal
+      save
+    end
+
+    LockedError = Class.new(::StandardError)
+
+    def retry!
+      raise LockedError if locked
+      self.status = 'pending'
+      signal
+      save!
+    end
+  end
+end

data/lib/jobs/migrate.rb

@@ -0,0 +1,19 @@
+class CreateJobs < ActiveRecord::Migration
+  def self.up
+    create_table :jobs, :force => true do |t|
+      t.string   :name,                          :null => false
+      t.text     :data
+      t.string   :status
+      t.datetime :created_at
+      t.datetime :updated_at
+      t.integer  :duration
+      t.integer  :taskable_id
+      t.string   :taskable_type
+      t.text     :details
+      t.boolean  :locked,     :default => false, :null => false
+      t.integer  :attempts,   :default => 0,     :null => false
+    end
+
+    add_index :jobs, [:status, :locked]
+  end
+end

data/lib/jobs/runnable.rb

@@ -0,0 +1,26 @@
+module Jobs
+  module Runnable
+    #
+    # returns a sql query condition given the included and excluded 
+    # jobs for the given process. e.g. jobs.name IN ('job1','job2') and jobs.name NOT IN ('jobs3','jobs5')
+    # would allow only job1 and job2 to run on the server, but also deny jobs3 and jobs5
+    # in this way the typical use would be to either use the deny or use the allow, but never both together...
+    # however the following will handle both...
+    #
+    def sql_runnable_conditions(allowed,denied)
+      conditions = "status='pending' and locked=0 "
+      if allowed and !allowed.empty? and denied and !denied.empty? # both have something
+        includes = allowed.map{|a| "'#{a}'" }.join(',')
+        conditions << %Q( and name IN (#{includes}) )
+        excludes = denied.map{|d| "'#{d}'" }.join(',')
+        conditions << %Q( and name not IN (#{excludes}) )
+      elsif allowed and !allowed.empty? and denied.nil? # only allowed
+        includes = allowed.map{|a| "'#{a}'" }.join(',')
+        conditions << %Q( and name IN (#{includes}) )
+      elsif denied and !denied.empty? and allowed.nil? # only denied
+        excludes = denied.map{|d| "'#{d}'" }.join(',')
+        conditions << %Q( and name not IN (#{excludes}) )
+      end
+    end
+  end
+end

data/lib/jobs/scheduler.rb

@@ -0,0 +1,41 @@
+require 'socket'
+require 'yaml'
+require 'active_record'
+require 'jobs/job'
+require 'jobs/config'
+
+module Jobs
+
+  module Scheduler
+    def schedule(job_name, options={})
+      threadable = options.delete(:threadable)
+      
+      job = Jobs::Job.new(:name          => job_name.to_s.strip,
+                          :data          => options,
+                          :taskable_id   => self.id,
+                          :taskable_type => self.class.to_s,
+                          :status        => "pending")
+      job.save!
+
+      signal(job_name,threadable)
+
+      job.id
+    end
+
+    #
+    # send a signal to wake job queue
+    #
+    # threadable: provides a hint that a threaded worker may be used to run the job
+    #
+    def signal(job_name=nil,threadable=nil)
+      socket = UDPSocket.open
+      #socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, true)
+      hosts = ::Jobs::Config.host_for_job(job_name)
+      if hosts.is_a?(Array)
+      else
+        socket.send(threadable ? 't' : 's', 0, hosts[:host], hosts[:port] )
+      end
+    end
+  end
+
+end

data/lib/jobs/server.rb

@@ -0,0 +1,382 @@
+#
+# Listen for UDP packets
+#
+# Start up X worker processes, communicates with worker processes by sending USR1 signal
+# and USR2 to kill the process
+#
+require 'yaml'
+require 'socket'
+require 'jobs/runnable'
+
+module Jobs
+
+  class Server
+    include Jobs::Runnable
+
+    def initialize(runpath, config_path, env)
+      @runpath = runpath
+      @config_path = File.expand_path(config_path)
+      @env = env
+      @workers = []
+      @next_worker = 0
+      load_config
+    end
+
+    def migrate
+      enable_logger
+      if @config['preload']
+        preload = @config['preload']
+        if preload.is_a?(Array)
+          preload.each { |f| require f }
+        else
+          require preload
+        end
+      end
+
+      require 'rubygems'
+      require 'active_record'
+
+      @logger.info("[job worker #{@pid}]: establish connection environment with #{@config_path.inspect} and env: #{@env.inspect}")
+      @db = YAML.load_file(File.join(File.dirname(@config_path),'database.yml'))[@env]
+      ActiveRecord::Base.establish_connection @db
+      ActiveRecord::Base.logger = @logger
+      #ActiveRecord::Base.logger = Logger.new( "#{@logfile}-db.log" )
+      #ActiveRecord::Base.logger = Logger.new( "/dev/null" )
+
+      # load the jobs/job model
+      require 'jobs/job'
+      require 'jobs/migrate'
+      CreateJobs.up
+    end
+
+    def kill
+      load_config
+      if File.exist?(@pid_file)
+        system("kill #{File.read(@pid_file)}")
+      else
+        puts "Job Queue Pid File not found! Try ps -ef | grep rjqueue"
+      end
+    end
+
+    def run(daemonize=true)
+
+      @daemonize = daemonize
+      if @daemonize
+        @child_up = false
+ 
+        # use this to determine when all workers are started and the child process is ready to listen for events
+        trap("USR1"){ @child_up = true }
+
+        if File.exist?(@pid_file)
+          STDERR.puts "Pid file for job already exists: #{@pid_file}"
+          exit 1
+        end
+        # daemonize, create a pipe to send status to the parent process, after the child has successfully started or failed
+        rd, wr = IO.pipe
+
+        @parent_pid = Process.pid
+
+        # wait for child process to start running before exiting parent process
+        fork do
+          rd.close
+          Process.setsid
+          fork do
+            begin
+              Process.setsid
+              File.open(@pid_file, 'wb') {|f| f << Process.pid}
+              Dir.chdir('/')
+              File.umask 0000
+              STDIN.reopen "/dev/null"
+              STDOUT.reopen "/dev/null", "a"
+              STDERR.reopen STDOUT
+
+              # XXX: change back to the runpath... this means if the runpath is removed
+              # say by a cap deploy... the daemon will likely die. 
+              Dir.chdir(@runpath)
+
+              startup
+
+              @logger.info "Job process active"
+
+              wr.write "Listening on udp://#{@host}:#{@port}\n"
+              wr.flush
+              wr.close # signal to our parent we're up and running, this lets the parent exit
+              run_loop
+            rescue => e
+              if wr.closed?
+                @logger.error "#{e.message} #{e.backtrace.join("\n")}"
+              else
+                wr.write e.message
+                wr.write e.backtrace.join("\n")
+                wr.write "\n"
+                wr.write "ERROR!"
+                wr.flush
+                wr.close
+              end
+            ensure
+              cleanup
+            end
+          end
+          wr.close
+        end
+        wr.close
+        output = rd.read
+        puts output
+        rd.close
+
+        w = 0
+        puts "Waiting for child workers to start..."
+        while( !@child_up and w < 20 ) do
+          sleep 5
+          w+= 1
+        end
+        if @child_up
+          puts "Workers alive"
+        else
+          puts "Check error log, workers may not be started, or you may be starting so many workers it has taken longer then 10 seconds to start them all. In either event checking the log for details would be the first place to look."
+          @parent_pid = nil
+        end
+
+        exit(1) if output.match(/ERROR/i)
+      else
+        startup
+        run_loop
+      end
+    end
+
+    private
+
+    def check_count
+      count = 0
+
+      query = "select count(id) from jobs where #{sql_runnable_conditions(@config['jobs_included'], @config['jobs_excluded'])}"
+      @logger.debug("timeout check: #{query.inspect}")
+
+      res = @conn.query(query)
+
+      count = res.fetch_row[0].to_i
+      count
+    rescue Mysql::Error => e
+      @logger.error("[jobqueue]: #{e.message}\n#{e.backtrace.join("\n")}")
+      count = 1
+      db_connect!
+      count
+    ensure
+      res.free if res
+    end
+
+    def db_connect!
+      # connect to the MySQL server
+      dbconf = YAML.load_file(File.join(File.dirname(@config_path),'database.yml'))[@env]
+      @conn = Mysql.real_connect(dbconf['host'], dbconf['username'], dbconf['password'], dbconf['database'], (dbconf['port'] or 3306) )
+    end
+
+    def run_loop
+
+      @sleep_time = @wait_time
+
+      @config['workers'].times {|i| start_worker(i) }
+
+      unless defined?(Jobs::Initializer) and Jobs::Initializer.ready?
+        require 'rubygems'
+        require 'jobs/client'
+        Jobs::Initializer.run! File.join(@runpath,@config['jobpath']), @config_path, @env
+      end
+
+      # job queue master process needs to be aware of the number of jobs pending on timeout
+      require 'rubygems'
+      require 'mysql'
+
+      db_connect!
+
+      begin
+        count = check_count
+        signal_work(count)
+
+        begin
+          Process.kill("USR1", @parent_pid) if !@parent_pid.nil?
+        rescue => e
+          @logger.error "#{e.message}\n#{e.backtrace.join("\n")}"
+        end
+
+        while(@running) do
+          count = 0
+          begin
+            if IO.select([@sock],[],[],@sleep_time)
+              while( (msg = @sock.read_nonblock(1)) ) do # each message is 1 byte ['s','t']
+                count += 1 # a new message, increment the request count
+              end
+            else
+              count = check_count
+            end
+          rescue Errno::EAGAIN => e
+            # there is more information pending, but we don't have it hear yet... not sure if this condition happens with UDP
+          end
+
+          if count > 0
+            @logger.info("[jobqueue]: received #{count} events")
+            # signal workers, distribute work load evenly to each worker
+            signal_work(count)
+          end
+
+        end
+      rescue Object => e
+        if @running
+          @logger.error "[jobqueue] #{e.message}\n#{e.backtrace.join("\n")}"
+          sleep 0.1 # throttle incase things get crazy
+          retry
+        end
+      end
+
+      @logger.info "Stopping workers"
+      @workers.each do|worker|
+        stop_worker(worker)
+      end
+
+    end
+
+    def stop_worker(pid)
+      @logger.info "Stopping worker: #{pid}"
+      Process.kill('USR2', pid)
+      Process.waitpid2(pid,0)
+    end
+
+    def start_worker(worker_id)
+      config      = @config
+      config_path = @config_path
+      logger      = @logger
+      env         = @env
+
+      rd, wr = IO.pipe
+      # startup workers
+      @logger.info "starting worker"
+      @workers << fork do
+        begin
+          rd.close
+          require 'jobs/worker'
+          worker = Jobs::Worker.new(config,@runpath, config_path,logger,env)
+          @logger.info "created worker: #{Process.pid}"
+          # drop a worker pid
+          File.open(@pid_file.gsub(/\.pid$/,"-#{worker_id}.pid"),'wb') {|f| f << Process.pid }
+          # signal parent
+          wr.write "up"
+          wr.close
+          worker.listen
+          # cleanup pid
+          File.unlink @pid_file.gsub(/\.pid$/,"-#{worker_id}.pid")
+        rescue Object => e
+          msg = "#{e.message}\n#{e.backtrace.join("\n")}"
+          if wr.closed?
+            @logger.error msg
+          else
+            wr.write msg
+            wr.close
+          end
+        end
+      end
+      wr.close
+      @logger.info "waiting for worker to reply"
+      msg = rd.read
+      rd.close
+      @logger.info "worker: #{msg.inspect}"
+      raise "Failed to start worker: #{msg.inspect}" if msg != 'up'
+    end
+
+    # tell a non-busy worker there is new work
+    def signal_work(count) # count is unused
+      return if count.nil? or count == 0
+      count.times do
+        Process.kill( 'USR1', @workers[@next_worker] )
+        @next_worker += 1 # simple round robin...
+        @next_worker = 0 if @next_worker >= @workers.size
+      end
+    end
+
+    def startup
+      enable_logger
+ 
+      # open sock connection 
+      @sock = bind_socket
+
+      @logger.info "Job process active"
+      @logger.info "Listening on udp://#{@host}:#{@port}\n"
+
+      @running = true
+
+      trap('TERM') { shutdown('TERM') }
+      trap('INT') { shutdown('INT') }
+      trap('HUP') { @logger.info 'ignore HUP' }
+    end
+ 
+    def shutdown(sig)
+      @logger.info "trap #{sig}"
+      @running = false # toggle the run state
+      trap(sig,"SIG_DFL") # turn the signal handler off
+      Process.kill("USR2", Process.pid) # resend the signal to trigger and exception in IO.select
+    end
+
+    def cleanup
+      @logger.info "[jobqueue] Stopping: #{@pid_file.inspect}"
+      if File.exist?(@pid_file)
+        File.unlink(@pid_file)
+      end
+      @conn.close if @conn
+    end
+
+    def bind_socket
+      # listen to udp packets
+      sock = UDPSocket.open
+      sock.bind(@host, @port)
+      sock
+    end
+
+    # load the server configuration, and intialize configuration instance variables
+    def load_config
+      @config = YAML.load_file(@config_path)
+      @config = @config[@env]
+      if @config['runpath']
+        runpath = @config['runpath']
+        if !runpath.match(/^\//)
+          @runpath = File.expand_path(runpath)
+        else
+          @runpath = runpath
+        end
+      end
+   
+      @pid_file = @config['pidfile'] or File.join(@runpath,'log','jobs.pid')
+
+      if @pid_file and !@pid_file.match(/^\//)
+        # make pidfile path absolute
+        @pid_file = File.join(@runpath,File.dirname(@pid_file), File.basename(@pid_file))
+      end
+
+      # store some common config keys
+      @wait_time     = @config['wait_time'] || 10
+      @port          = @config['port'] || 4321
+      @host          = @config['host'] || '127.0.0.1'
+      @threads       = @config['threads'] || 10
+    end
+
+    # setup logging
+    def enable_logger
+      require 'logger'
+      if @config['logfile']
+        @logfile = @config['logfile']
+        if @daemonize
+          if !@logfile.match(/^\//)
+            @logfile = File.join(@runpath, @logfile)
+          end
+          @logger = Logger.new( @logfile )
+        else
+          @logger = Logger.new(STDOUT)
+        end
+      else
+        @logger = Logger.new( '/dev/null' )
+      end
+
+      @logger.level = Logger::ERROR if @env == 'production'
+    end
+
+  end
+
+end