pipa-monkeyjob 0.1.2 → 0.1.3

data/README.rdoc

@@ -2,7 +2,7 @@
 
 
 === Installation
-  gem install pipa-monkeyjob -s gems.github.com
+  gem install pipa-monkeyjob --source http://gems.github.com
 
 === Usage
 

data/lib/monkeyjob.rb

@@ -1,11 +1,15 @@
 require 'activesupport'
 
 module Monkey
+  mattr_accessor :logger
+  @@logger = Logger.new(STDERR)
+
   autoload :Client, 'monkeyjob/client'
   autoload :Broker, 'monkeyjob/broker'
   autoload :Runner, 'monkeyjob/runner'
   autoload :Juggler, 'monkeyjob/juggler'
 end
 
+require 'monkeyjob/logger'
 require 'monkeyjob/job'
 

data/lib/monkeyjob/broker/active_record.rb

@@ -3,51 +3,80 @@ require 'monitor'
 begin
   require 'activerecord'
 rescue MissingSourceFile => e
-  raise e.exception('Please install ActiveRecord!')
+  raise $!.exception('missing activerecord!')
 end
 
 module Monkey
   class Broker
     class ActiveRecord < Broker
-      def initialize
-        extend MonitorMixin
+      class Cookie < Job::Cookie::Impl
+        attr_reader :queue, :id
+      
+        def initialize(broker, jm)
+          super(jm.handler)
+          @broker, @queue, @id = broker, jm.queue, jm.id
+        end
         
-        @callback = proc do |job, id|
-          synchronize do
-            JobModel.delete(id)
-            @jobs.delete(id)
-          end
+        def client
+          @broker.client
+        end
+        
+        def to_s
+          "#{id}#{queue && '@' + queue} #{job}"
+        end
+        
+        private
+        def on_state_changed(was)
+          super
+          @broker.send(:on_finish, self) if finished?
         end
       end
-
-      def start(&callback)
-        @poller = Thread.new { poll(&callback) }
-      end
-
-      def close
-        @dead = true
-        @poller.run
+    
+      attr_reader :client
+    
+      def initialize(options = {})
+        extend MonitorMixin
+        @queues = Array(options[:queue] || options[:queues])
+        @client = Client::ActiveRecord.new
       end
 
-      private
-      def poll
+      def start(&callback)
         @jobs = Set.new
         loop do
-          JobModel.transaction do
-            jms = JobModel.candidates :lock => true
-            synchronize do
-              for jm in jms
-                @jobs << jm.id if !@jobs.include?(jm.id) && yield(Job.new(jm.handler_object, jm.id, &@callback))
+          synchronize do
+            JobModel.logger.hush do |old_logger_level|
+              JobModel.transaction do
+                for jm in JobModel.candidates(@queues).all(:lock => true)
+                  JobModel.logger.with_level(old_logger_level) do
+                    if @jobs.add?(jm.id) && !callback.call(Cookie.new(self, jm))
+                      @jobs.delete(jm.id)
+                      break
+                    end
+                  end
+                end
+                JobModel.update_all({:run_at => 5.minutes.from_now}, :id => @jobs.to_a)
               end
-              JobModel.update_all({:run_at => 5.minutes.from_now}, :id => @jobs.to_a)
             end
           end
-          break if @dead
           sleep(5)
-          break if @dead
         end
-      rescue => e
-        puts e, e.backtrace
+      end
+      
+      def close
+        self
+      end
+
+      private
+      def on_finish(cookie)
+        synchronize do
+          @jobs.delete(cookie.id)
+          if cookie.success?
+            JobModel.delete(cookie.id)
+            Monkey.logger.debug "Deleted job #{cookie}"
+          end
+        end
+      rescue
+        Monkey.logger.error $!
       end
     end
   end

data/lib/monkeyjob/broker/active_record/job_model.rb

@@ -2,17 +2,13 @@ module Monkey
   class Broker
     class ActiveRecord
       class JobModel < ::ActiveRecord::Base
-        extend ::ActiveSupport::Memoizable
-
         set_table_name 'monkey_jobs'
+        serialize :handler
 
-        named_scope :candidates, :conditions => ['run_at <= ?', Time.now]
-        
-        def handler_object
-          YAML.load(handler)
-        end
-        
-        memoize :handler_object
+        named_scope :candidates, proc {|queues|
+          {:conditions => ["#{queues.blank? ? 'queue IS NULL' : 'queue IN (:queues)'} AND run_at <= :now",
+          {:now => Time.now, :queues => queues}], :order => 'run_at, id'}
+        }
       end
     end
   end

data/lib/monkeyjob/client.rb

@@ -1,6 +1,8 @@
 module Monkey
   class Client
     autoload :ActiveRecord, 'monkeyjob/client/active_record'
+    
+    cattr_accessor :default
   end
 end
 

data/lib/monkeyjob/client/active_record.rb

@@ -1,8 +1,20 @@
 module Monkey
   class Client
-    class ActiveRecord
-      def submit(handler, options = {})
-        Broker::ActiveRecord::JobModel.create(:run_at => options[:run_at] || Time.now, :handler => handler.to_yaml)
+    class ActiveRecord < Client
+      def initialize(options = {})
+        @queue = options[:queue]
+      end
+    
+      def submit(job, options = {})
+        jm = Broker::ActiveRecord::JobModel.silence do
+          Broker::ActiveRecord::JobModel.new {|m|
+            m.run_at = options[:run_at] || Time.now
+            m.handler = job
+            m.queue = options.fetch(:queue, @queue)
+          }.tap {|jm| jm.save! }
+        end
+        Monkey.logger.info { "Submitted #{job} to #{jm.queue}:#{jm.id}" }
+        self
       end
     end
   end

data/lib/monkeyjob/job.rb

@@ -1,20 +1,102 @@
 module Monkey
   class Job
-    attr_reader :handler, :result, :exception
+    def to_proc
+      method(:run).to_proc
+    end
+    
+    def to_s
+      self.class.name
+    end
+    
+    def submit(options = {})
+      self.class.submit(self, options)
+    end
 
-    def initialize(handler, *callback_args, &callback)
-      @handler, @callback, @callback_args = handler, callback, callback_args
+    def submit_to(client, options = {})
+      client.submit(self, options)
     end
 
-    def call
-      @result = handler.run
-    rescue => @exception
-    ensure
-      @callback.call(self, *@callback_args)
+    def self.submit(handler, options = {})
+      Client.default.submit(handler, options)
     end
-    
-    def to_proc
-      method(:call).to_proc
+
+    class Cookie
+      attr_reader :state, :result, :exception, :started_at, :finished_at
+
+      def running?
+        state == :running
+      end
+
+      def success?
+        state == :success
+      end
+
+      def error?
+        state == :error
+      end
+      
+      def finished?
+        success? || error?
+      end
+
+      private
+      def start!
+        Monkey.logger.info "Running #{self}"
+        @result, @exception, @started_at, @finished_at, self.state = nil, nil, Time.now, nil, :running
+      end
+
+      def unstart!
+        @result, @exception, @started_at, @finished_at, self.state = nil, nil, nil, nil, nil
+      end
+
+      def success!(result = nil)
+        @result, @exception, @finished_at, self.state = result, nil, Time.now, :success
+        Monkey.logger.info "Job #{self} succeeded after #{finished_at - started_at} sec."
+      end
+
+      def error!(exception = nil)
+        @result, @exception, @finished_at, self.state = nil, exception, Time.now, :error
+        Monkey.logger.error(exception) if exception
+        Monkey.logger.error "Job #{self} failed after #{finished_at - started_at} sec."
+      end
+      
+      attr_writer :state
+
+      class Impl < Cookie
+        attr_reader :job
+        
+        def initialize(job)
+          @job = job
+        end
+        
+        def to_s
+          job.to_s
+        end
+
+        def to_proc
+          method(:run).to_proc
+        end
+        
+        def submit(job, options = {})
+          client.submit(job, {:queue => queue}.update(options))
+        end
+        
+        private
+        def state=(v)
+          was = state
+          super.tap { on_state_changed(was) }
+        end
+
+        def on_state_changed(was)
+        end
+
+        def run
+          start!
+          success!(job.to_proc.call(self))
+        rescue
+          error!($!)
+        end
+      end
     end
   end
 end

data/lib/monkeyjob/juggler.rb

@@ -1,19 +1,25 @@
 module Monkey
   class Juggler
-    def initialize
-      @broker = Broker::ActiveRecord.new
-      @runner = Runner::ThreadPool.new
+    def initialize(options = {})
+      @broker = options[:broker] || raise(ArgumentError ':broker is mandatory')
+      @runner = options[:runner] || raise(ArgumentError ':runner is mandatory')
     end
     
     def start
-      @broker.start do |job|
-        @runner.run(job)
-      end
+      @broker.start {|cookie| @runner.run?(cookie) }
+      self
+    end
+    
+    def run
+      start
+    ensure
+      close
     end
     
     def close
       @broker.close
       @runner.close
+      self
     end
   end
 end

data/lib/monkeyjob/logger.rb

@@ -0,0 +1,33 @@
+module Monkey
+  module LoggerPatch
+#    def included?(other)
+#      other.send :alias_method, :global_level, :level
+#      other.send :alias_method, :global_level=, :level=
+#    end
+
+#    def level
+#      Thread.current["logger[#{object_id}].level"] || global_level
+#    end
+
+#    def level=(level)
+#      Thread.current["logger[#{object_id}].level"]
+#    end
+    
+    def with_level(temp_level)
+      return yield old_level unless silencer
+      begin
+        old_level, self.level = self.level, temp_level
+        yield old_level
+      ensure
+        self.level = old_level
+      end
+    end
+
+    def hush(temp_level = Logger::ERROR, &block)
+      with_level(temp_level, &block)
+    end
+  end
+  
+  [Logger, ActiveSupport::BufferedLogger].each {|m| m.send :include, LoggerPatch }
+end
+

data/lib/monkeyjob/runner.rb

@@ -1,6 +1,10 @@
 module Monkey
   class Runner
+    autoload :Fork, 'monkeyjob/runner/fork'
     autoload :ThreadPool, 'monkeyjob/runner/thread_pool'
+    
+    def close
+    end
   end
 end
 

data/lib/monkeyjob/runner/fork.rb

@@ -0,0 +1,85 @@
+require 'monitor'
+
+module Monkey
+  class Runner
+    class Fork < Runner
+      autoload :Rails, 'monkeyjob/runner/fork/rails'
+
+      def initialize(options = {})
+        extend MonitorMixin
+        @max = options[:max] || 4
+        @workers = Set.new
+      end
+      
+      def run(cookie)
+        run_core(:run_local, cookie)
+      end
+      
+      def run?(cookie)
+        run_core(:run_local?, cookie)
+      end
+
+      def run_local(*args)
+        raise ArgumentError, 'block must be provided' unless block_given?
+        pid = fork do
+          trap('TERM') do
+            Monkey.logger.warn "Process #{Process.pid} received SIGTERM, will exit!"
+            exit!(2)
+          end
+          ActiveRecord::Base.spawn_reconnect
+          begin
+            yield(*args)
+            exit!(0)
+          rescue
+            Monkey.logger.error $!
+            exit!(1)
+          end
+        end
+        (spid = Process.detach(pid))[:pid] = pid
+        spid
+      end
+
+      def run_local?(*args, &block)
+        synchronize do
+          return nil unless @workers.size < @max
+          @sweeper ||= Thread.new { loop { sleep(1); synchronize { sweep } } }
+          run_local(*args, &block).tap {|spid| @workers << spid }
+        end
+      end
+
+      def close
+        synchronize do
+          @sweeper.try :kill
+          @workers.each do |spid|
+            begin
+              Monkey.logger.warn { "Killing process #{spid[:pid]}" }
+              Process.kill('TERM', spid[:pid])
+            rescue
+              Monkey.logger.debug $!
+            end
+          end
+        end
+      end
+      
+      private
+      def run_core(method, cookie)
+        synchronize do
+          send(method, cookie, &cookie.job).tap do |spid|
+            (spid[:cookie] = cookie).send(:start!) if spid
+          end
+        end
+      end
+      
+      def sweep
+        @workers.delete_if {|spid| finish!(spid) }
+      rescue
+        Monkey.logger.error($!)
+      end
+      
+      def finish!(spid)
+        !spid.alive?.tap {|alive| spid[:cookie].try(spid.value.success? ? :success! : :error!) unless alive }
+      end
+    end
+  end
+end
+

data/lib/monkeyjob/runner/fork/rails.rb

@@ -0,0 +1,32 @@
+module Monkey
+  class Runner
+    class Fork
+      class Rails < Fork
+        def run_local(*args)
+          super(*args) do
+            begin
+              ActiveRecord::Base.dirty_reconnect!
+              yield(*args)
+            ensure
+              ActiveRecord::Base.connection_handler.clear_all_connections!
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+class Monkey::Runner::Fork::Rails
+  module ActiveRecord
+    module BasePatch
+      def dirty_reconnect!
+        @@connection_handler = ActiveRecord::ConnectionAdapters::ConnectionHandler.new
+        establish_connection
+      end
+    end
+  end
+end
+
+::ActiveRecord::Base.extend Monkey::Runner::Fork::Rails::ActiveRecord::BasePatch
+

data/lib/monkeyjob/runner/thread_pool.rb

@@ -3,17 +3,32 @@ require 'threadpool'
 module Monkey
   class Runner
     class ThreadPool < Runner
-      def initialize
-        @pool = ::ThreadPool.new
+      def initialize(options = {})
+        @pool = ::ThreadPool.new(options)
+      end
+      
+      def run(cookie)
+        run_local(&cookie)
       end
 
-      def close
-        @pool.close
+      def run?(cookie)
+        run_local?(&cookie)
+      end
+
+      def run_local?(*args, &block)
+        @pool.run(*args, &block)
+        self
       end
 
-      def run(job)
-        @pool.try_run(&job)
+      def run_local?(*args, &block)
+        @pool.run?(*args, &block) && self
+      end
+
+      def close
+        @pool.close
+        self
       end
     end
   end
 end
+

data/rails/init.rb

@@ -0,0 +1,2 @@
+Monkey.logger = RAILS_DEFAULT_LOGGER
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: pipa-monkeyjob
 version: !ruby/object:Gem::Version 
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors: 
 - Igor Gunko
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-08 00:00:00 -07:00
+date: 2009-06-17 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,26 +22,6 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 2.2.2
     version: 
-- !ruby/object:Gem::Dependency 
-  name: activerecord
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 2.2.2
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: pipa-threadpool
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.2.1
-    version: 
 - !ruby/object:Gem::Dependency 
   name: thoughtbot-shoulda
   type: :development
@@ -77,6 +57,7 @@ files:
 - Rakefile
 - lib/monkeyjob.rb
 - lib/pipa-monkeyjob.rb
+- lib/monkeyjob/logger.rb
 - lib/monkeyjob/job.rb
 - lib/monkeyjob/juggler.rb
 - lib/monkeyjob/client.rb
@@ -86,6 +67,9 @@ files:
 - lib/monkeyjob/broker/active_record.rb
 - lib/monkeyjob/broker/active_record/job_model.rb
 - lib/monkeyjob/runner/thread_pool.rb
+- lib/monkeyjob/runner/fork.rb
+- lib/monkeyjob/runner/fork/rails.rb
+- rails/init.rb
 has_rdoc: true
 homepage: http://github.com/pipa/monkeyjob
 post_install_message: