taf2-rjqueue 0.1.4

data/lib/jobs/worker.rb

@@ -0,0 +1,225 @@
+#
+# Worker Process
+# 
+# Loads whatever environment you need running to manage and run jobs.
+#
+# A master process listens for UDP Packets and then delegates to a worker process
+# The master keep track of how busy the worker processes are.
+#
+# Besides monitoring the child processes are forked by the master process. The master process
+# can then use signals to trigger the worker to check for new work.
+#
+#
+# pid = fork do
+#   worker = Worker.new(config,logger)
+#   signal our parent to let it know we're up and running
+#   worker.listen # waits for SIGUSR1 to check for work, and SIGUSR2 to exit
+# end
+#
+# wait for child to signal us with it's startup status
+#
+# workers << pid
+#
+
+require 'thread'
+require 'rubygems'
+require 'mysql'
+require 'active_record'
+require 'jobs/client'
+require 'jobs/runnable'
+
+module Jobs
+  class Worker
+    include Jobs::Runnable
+    def initialize(config, runpath, config_path, logger, env)
+      @config = config
+      @runpath = runpath
+      @config_path = config_path
+      @logger = logger
+      @queue = Queue.new
+      @lock = Mutex.new
+      @sig_lock = Mutex.new
+      @thread = nil
+      @env = env
+      preload
+
+      unless defined?(Jobs::Initializer) and Jobs::Initializer.ready?
+        Jobs::Initializer.run! File.join(@runpath,@config['jobpath']), @config_path, @env
+      end
+
+      establish_connection
+      @threads = (@config['threads'] or 1).to_i
+      @pid = Process.pid
+    end
+
+    def listen
+      @alive = true
+      @thread = Thread.new{ worker }
+      @count = 0
+      trap("USR1") { @sig_lock.synchronize {@count += 1} } # queue up some work 
+      trap("USR2") { @alive = false; trap("USR2",'SIG_DFL') } # sent to stop normally
+      trap('TERM') { @alive = false; trap("TERM",'SIG_DFL') }
+      trap('INT')  { @alive = false; trap("INT",'SIG_DFL') }
+
+      while @alive do
+        sleep 1 # 1 second resolution
+        count = 0
+        @sig_lock.synchronize{ count = @count; @count = 0 }
+        if count > 0
+          @logger.debug("[job worker #{@pid}]: processing #{count} jobs")
+          if count > @threads # we've queued up more then we can handle chunk it out and start chugging away
+            (count/@threads).times do
+              a = count - (count-@threads)
+              #@logger.debug("[job worker #{@pid}]: queueing #{a}")
+              @queue << a
+              count = (count-@threads)
+            end
+            if count > 0
+              #@logger.debug("[job worker #{@pid}]: queueing #{count}")
+              @queue << count
+            end
+          else
+            #@logger.debug("[job worker #{@pid}]: queueing #{count}")
+            @queue << count
+          end
+        end
+      end
+      @queue << nil
+      @logger.info "[job worker #{@pid}]: Joining with main thread"
+      @thread.join # wait for the worker
+    end
+
+  private
+    def worker
+      while @alive
+        count = 0
+
+        if @queue.empty?
+          @logger.debug "[job worker #{@pid}]: #{Process.pid} waiting..."
+          count = @queue.pop # sleep until we get some work
+        else
+          begin # pop as many as we can until we reach a max
+            while count < @threads do
+              value = @queue.pop(true)
+              if value.nil?
+                count = nil
+                break
+              end
+              count += value
+            end
+          rescue ThreadError => e
+          end
+        end
+
+        @logger.debug "[job worker #{@pid}]: #{@pid} awake with: #{count} suggested jobs"
+
+        break if count.nil? # if we get a nil count we're done
+
+        count = @threads if count == 0
+
+        jobs = []
+
+        begin
+          Jobs::Job.transaction do
+
+            conditions = sql_runnable_conditions(@config['jobs_included'], @config['jobs_excluded'])
+            jobs = Jobs::Job.find(:all, :conditions => conditions, :limit => count ) # only retrieve as many jobs as we were told about
+
+            # lock the jobs we're about to process here
+            @logger.debug "[job worker #{@pid}]: got #{jobs.size} to process out of #{count}"
+            jobs.each do|job|
+              job.locked = true
+              job.save!
+            end
+          end
+
+          reload!
+
+          if jobs.size > 2 and @threads > 1
+            threads = jobs.map do |job|
+              Thread.new(job) do|j|
+                process(j)
+              end
+            end
+            threads.each{ |t| t.join }
+          else
+            jobs.each do |job|
+              process(job)
+              break unless @alive
+            end
+          end
+        ensure
+          Jobs::Job.transaction do
+            jobs.each do|job|
+              if job.locked
+                job.locked = false
+                job.save!
+              end
+            end
+          end
+        end
+
+      end
+      @logger.info "[job worker #{@pid}]: exiting work thread"
+    end
+
+    def process(job)
+      job.status = 'processing'
+      job.save!
+      task = job.instance(@logger, @lock)
+      if task
+        task.real_execute
+      else
+        # report no job defined error
+      end
+    rescue Object => e
+      @logger.error "[job worker #{@pid}]: #{e.message}\n#{e.backtrace.join("\n")}"
+      if job
+        job.details ||= ""
+        job.details << "#{e.message}\n#{e.backtrace.join("\n")}"
+        job.status = 'error'
+      end
+    end
+
+    def preload
+      if @config['preload']
+        preload = @config['preload']
+        if preload.is_a?(Array)
+          preload.each { |f| require f }
+        else
+          require preload
+        end
+      end
+    end
+
+    def establish_connection
+
+      @logger.info("[job worker #{@pid}]: establish connection environment with #{@config_path.inspect} and env: #{@env.inspect}")
+      @db = YAML.load_file(File.join(File.dirname(@config_path),'database.yml'))[@env]
+      ActiveRecord::Base.establish_connection @db
+      ActiveRecord::Base.logger = @logger
+
+      # load the jobs/job model
+      require 'jobs/job'
+    end
+
+    def reload!
+      if defined?(RAILS_ENV) and RAILS_ENV != 'production'
+        # if this method is defined, it'll be called before each job is executed... during
+        # development this allows you to change your job code without restarting the job server...
+        # clear things out
+
+        ActiveRecord::Base.reset_subclasses if defined?(ActiveRecord)
+        ActiveSupport::Dependencies.clear if defined?(ActiveSupport)
+        ActiveRecord::Base.clear_reloadable_connections! if defined?(ActiveRecord)
+        
+        # reload the environment... XXX: skipping the run_callbacks from reload_application in dispatcher.rb action_pack...
+        # this might cause the shit to hit the fan...
+        Routing::Routes.reload if defined?(Routing)
+        ActionController::Base.view_paths.reload! if defined?(ActionController)
+        ActionView::Helpers::AssetTagHelper::AssetTag::Cache.clear if defined?(ActionView)
+      end
+    end
+
+  end
+end

data/tests/jobs/find_file_job.rb

@@ -0,0 +1,10 @@
+class FindFileJob < Jobs::Base
+  # simple test to find a file
+
+  def execute
+    File.open("#{File.dirname(__FILE__)}/output", "w") do|f|
+      f << call("find #{File.dirname(__FILE__)} -name find_file_job.rb").inspect
+    end
+  end
+
+end

data/tests/jobs/image_thumb_job.rb

@@ -0,0 +1,28 @@
+require 'RMagick'
+require 'jobs/client'
+
+# require test models
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+require 'image'
+
+class ImageThumbJob < Jobs::Base
+  def execute
+    #@logger.debug("lookup record: #{@record.data[:id].inspect} #{Image.connection.inspect}")
+    # fetch the record to process
+    image_record = Image.find_by_id( @record.data[:id].to_i )
+
+    # read the image into memory
+    image = Magick::Image.read(image_record.path).first
+
+    # scale the image to the given dimensions
+    image.change_geometry!( @record.data[:size] ) { |cols,rows,img|
+      img.resize!(cols < 1 ? 1 : cols, rows < 1 ? 1 : rows)
+    }
+
+    # save the thumbnail, where we can find it later
+    image.write image_record.thumb_path
+
+    # save the record
+    image_record.save
+  end
+end

data/tests/jobs/simple_job.rb

@@ -0,0 +1,6 @@
+
+class SimpleJob < Jobs::Base
+  def execute
+    sleep 1 # one second
+  end
+end

data/tests/lib/create_test_data.rb

@@ -0,0 +1,17 @@
+class CreateTestData < ActiveRecord::Migration
+
+  def self.up
+    create_table :messages, :force => true do |t|
+      t.string :subject
+      t.string :content
+      t.integer :image_id
+    end
+
+    create_table :images, :force => true do |t|
+      t.string :name
+      t.string :alt_text
+      t.string :path
+    end
+  end
+
+end

data/tests/lib/image.rb

@@ -0,0 +1,18 @@
+class Image < ActiveRecord::Base
+  include Jobs::Scheduler
+
+  has_one :job, :class_name => 'Jobs::Job', :as => :taskable, :dependent => :destroy
+
+  after_create :filter_image
+
+  def filter_image
+    # schedule the attached image to be scaled to 64x32
+    schedule(:image_thumb,{:id => self.id, :size => '64x32' })
+  end
+
+  def thumb_path
+    extname = File.extname(self.path)
+    pathname = self.path.gsub(/#{extname}$/,'')
+    "#{pathname}-thumb#{extname}"
+  end
+end

data/tests/lib/message.rb

@@ -0,0 +1,3 @@
+class Message < ActiveRecord::Base
+  belongs_to :image
+end

data/tests/test_server.rb

@@ -0,0 +1,86 @@
+require 'test/unit'
+require 'rubygems'
+require 'active_record'
+
+# load the jobs client environment
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+require 'jobs/client'
+
+# load test models
+$:.unshift File.join(File.dirname(__FILE__),'lib')
+require 'message'
+require 'image'
+
+Jobs::Initializer.test!
+
+class TestServer < Test::Unit::TestCase
+
+  def setup
+    # ensure we have a connection established
+    if not ActiveRecord::Base.connected?
+      ActiveRecord::Base.establish_connection YAML.load_file(File.join(File.dirname(__FILE__),'..','config','database.yml'))['test']
+      ActiveRecord::Base.logger = Logger.new(File.join(File.dirname(__FILE__),'logs','test-db.log'))
+    end
+  end
+
+  #
+  # test running a background thumbnail creation job
+  #
+  def test_single_image_thumb_job
+    timer = Time.now
+    # create new message record
+    image = Image.new :name => 'test',
+                      :alt_text => 'sample',
+                      :path => File.expand_path(File.join(File.dirname(__FILE__),'sample.png'))
+
+    # save the new record, should trigger the image
+    assert image.save
+
+    until image.job.status != 'processing' and image.job.status != 'pending'
+      sleep 1
+      image = Image.find_by_id(image.id)
+      puts "waiting for job to complete... '#{image.job.status}' and #{Jobs::Job.count(:conditions => ["status = 'complete'"])} completed"
+    end
+
+    image = Image.find_by_id(image.id)
+
+    assert_equal 'complete', image.job.status
+    assert File.exist?(image.thumb_path)
+    dur = Time.now - timer
+    puts "Duration: #{image.job.updated_at.to_f - image.job.created_at.to_f} seconds and duration: #{image.job.duration}, real: #{dur}"
+  end
+
+  def test_high_load_thumb_jobs
+
+    images = []
+
+    20.times do
+      # create new message record
+      image = Image.new :name => 'test',
+                        :alt_text => 'sample',
+                        :path => File.expand_path(File.join(File.dirname(__FILE__),'sample.png'))
+
+      # save the new record, should trigger the image
+      assert image.save
+      images << image
+    end
+
+    images.each do|image|
+
+      until image.job.status != 'processing' and image.job.status != 'pending'
+        sleep 1 # poll every second for updated job status
+        image = Image.find_by_id(image.id)
+        puts "waiting for job to complete... '#{image.job.status}' and #{Jobs::Job.count(:conditions => ["status = 'complete'"])} completed"
+      end
+
+      assert_equal 'complete', image.job.status
+      assert File.exist?(image.thumb_path)
+
+    end
+  end
+ 
+  def test_file_search_job
+    puts "call test_file_search_job"
+  end
+
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification 
+name: taf2-rjqueue
+version: !ruby/object:Gem::Version 
+  version: 0.1.4
+platform: ruby
+authors: 
+- Todd A. Fisher
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-12-15 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: A Job Queue Server.  Responses to UDP requests.
+email: todd.fisher@gmail.com
+executables: 
+- rjqueue
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- LICENSE
+- README
+- Rakefile
+- bin/rjqueue
+- config/database.yml
+- config/jobs.yml
+- lib/jobs/scheduler.rb
+- lib/jobs/initializer.rb
+- lib/jobs/config.rb
+- lib/jobs/migrate.rb
+- lib/jobs/worker.rb
+- lib/jobs/runnable.rb
+- lib/jobs/job.rb
+- lib/jobs/server.rb
+- lib/jobs/client.rb
+- lib/jobs/base.rb
+has_rdoc: true
+homepage: http://idle-hacking.com/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: rjqueue
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Ruby Job Queue
+test_files: 
+- tests/sample.png
+- tests/test_server.rb
+- tests/jobs/simple_job.rb
+- tests/jobs/image_thumb_job.rb
+- tests/jobs/find_file_job.rb
+- tests/lib/message.rb
+- tests/lib/image.rb
+- tests/lib/create_test_data.rb