recurrent 0.3.0 → 0.4.0

data/.gitignore

@@ -1,6 +1,5 @@
 .DS_Store
 coverage
-.rvmrc
 Gemfile.lock
 .bundle
 *.gem

data/.rvmrc

@@ -0,0 +1 @@
+rvm 1.8.7@recurrent

data/README.markdown

@@ -126,7 +126,7 @@ To limit the number of tasks that run simultaneously:
 
     configure.maximum_concurrent_tasks = 5
 
-This will run up to the above number of tasks simultaneously, other tasks will wait until a slot opens up before executing. Tasks that run more frequently, e.g. once a minute, will have precedence over tasks that run once an hour or day etc.
+This will run up to the above number of tasks simultaneously, if there are too many tasks running it will skip any extra tasks for that run. Tasks that run less frequently are prioritized. For example if you can run a maximum of 5 tasks, 4 of which are currently running, and you are now scheduled to run two tasks, one that runs once an hour, and one that runs once a minute, the hourly task will get the slot and the minutely task will not run until its next scheduled run.
 
 
 

data/bin/recurrent

@@ -8,6 +8,9 @@ opts = Trollop::options do
   opt :ruby, "Ruby code to execute", :type => :string
   opt :system, "Command to send to the command line", :type => :string
   opt :name, "Name of the task", :default => 'command_line'
+  opt :maximum_concurrent_tasks, "Maximum number of concurrent tasks to run", :type => :integer
+  opt :pool_size, "Size of database connection pool", :type => :integer
+  opt :locker_pool_size, "Size of locker database connection pool (should be >= maximum_concurrent_tasks)", :type => :integer
 end
 
 begin

data/lib/recurrent/configuration.rb

@@ -3,7 +3,7 @@ module Recurrent
 
     class << self
 
-      attr_accessor :logging, :wait_for_running_tasks_on_exit_for, :maximum_concurrent_tasks
+      attr_accessor :logging, :wait_for_running_tasks_on_exit_for, :maximum_concurrent_tasks, :pool_size, :locker_pool_size
 
       def self.block_accessor(*fields)
         fields.each do |field|
@@ -19,7 +19,7 @@ module Recurrent
           ")
         end
       end
-      block_accessor :logger, :save_task_schedule, :load_task_schedule, :save_task_return_value, :load_task_return_value, :process_locking, :handle_slow_task, :setup
+      block_accessor :logger, :save_task_schedule, :load_task_schedule, :save_task_return_value, :load_task_return_value, :process_locking, :task_locking, :handle_slow_task, :setup
 
     end
   end

data/lib/recurrent/scheduler.rb

@@ -94,7 +94,8 @@ module Recurrent
                       :action => block,
                       :save => options[:save],
                       :logger => logger,
-                      :scheduler => self)
+                      :scheduler => self,
+                      :disable_task_locking => options[:disable_task_locking])
       @tasks.add_or_update(task)
       logger.info "| #{key} added to Scheduler"
     end

data/lib/recurrent/task.rb

@@ -11,52 +11,54 @@ module Recurrent
       @save = options[:save]
       @logger = options[:logger]
       @scheduler = options[:scheduler]
+      @disable_task_locking = options[:disable_task_locking]
       Configuration.save_task_schedule.call(name, schedule) if Configuration.save_task_schedule
     end
 
     def execute(execution_time)
       return handle_still_running(execution_time) if running?
+      return if Configuration.maximum_concurrent_tasks.present? && (scheduler.executing_tasks >= Configuration.maximum_concurrent_tasks)
       @thread = Thread.new do
         Thread.current["execution_time"] = execution_time
+        scheduler && scheduler.increment_executing_tasks
         begin
-          if Configuration.maximum_concurrent_tasks.present?
-            limit_execution_to_max_concurrency
-          else
-            call_action
-          end
-        rescue TooManyExecutingTasks
-          scheduler.decrement_executing_tasks
-          sleep(0.1)
-          retry
+          call_action(execution_time)
         rescue => e
           logger.warn("#{name} - #{e.message}")
           logger.warn(e.backtrace)
+        ensure
+          scheduler && scheduler.decrement_executing_tasks
         end
       end
     end
 
-    def limit_execution_to_max_concurrency
-      if (scheduler.increment_executing_tasks <= Configuration.maximum_concurrent_tasks) && task_is_next_in_line?
-        call_action
-        scheduler.decrement_executing_tasks
-      else
-        raise TooManyExecutingTasks
-      end
-    end
-
-    def task_is_next_in_line?
-      self == scheduler.tasks.next_for_execution_at_time(Thread.current["execution_time"])
-    end
+    def call_action(execution_time=nil)
+      if Configuration.task_locking && !@disable_task_locking
+        logger.info "#{name} - #{execution_time.to_s(:seconds)}: attempting to establish lock"
+        lock_established = Configuration.task_locking.call(name) do
+          if Configuration.load_task_return_value && action.arity == 1
+            previous_value = Configuration.load_task_return_value.call(name)
 
-    def call_action
-      if Configuration.load_task_return_value && action.arity == 1
-        previous_value = Configuration.load_task_return_value.call(name)
+            return_value = action.call(previous_value)
+          else
+            return_value = action.call
+          end
+          save_results(return_value) if save?
 
-        return_value = action.call(previous_value)
+          # If a task finishes quickly hold the lock for a few seconds to avoid releasing it before other processes try to pick up the task
+          sleep(1) until Time.now - execution_time > 5 if execution_time
+        end
+        logger.info "#{name} - #{execution_time.to_s(:seconds)}: locked by another process" unless lock_established
       else
-        return_value = action.call
+        if Configuration.load_task_return_value && action.arity == 1
+          previous_value = Configuration.load_task_return_value.call(name)
+
+          return_value = action.call(previous_value)
+        else
+          return_value = action.call
+        end
+        save_results(return_value) if save?
       end
-      save_results(return_value) if save?
     end
 
     def handle_still_running(current_time)

data/lib/recurrent/version.rb

@@ -1,3 +1,3 @@
 module Recurrent
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/recurrent/worker.rb

@@ -4,6 +4,9 @@ module Recurrent
     attr_accessor :scheduler, :logger
 
     def initialize(options={})
+      Configuration.maximum_concurrent_tasks = options[:maximum_concurrent_tasks]
+      Configuration.pool_size = options[:pool_size]
+      Configuration.locker_pool_size = options[:locker_pool_size]
       Configuration.setup.call if Configuration.setup
       file = options[:file]
       @scheduler = Scheduler.new(file)
@@ -41,7 +44,7 @@ module Recurrent
     def execute
       loop do
         execution_time = scheduler.tasks.next_execution_time
-        tasks_to_execute = scheduler.tasks.scheduled_to_execute_at(execution_time, :sort_by_frequency => !!Configuration.maximum_concurrent_tasks)
+        tasks_to_execute = scheduler.tasks.scheduled_to_execute_at(execution_time, :sort_by_frequency => !!Configuration.maximum_concurrent_tasks).reverse
 
         wait_for_running_tasks && break if $exit
 

data/spec/task_spec.rb

@@ -222,14 +222,15 @@ module Recurrent
         @task1 = Task.new(:name => 'task1',
                          :scheduler => scheduler,
                          :schedule => schedule.clone,
-                         :action => lambda { sleep(1)})
+                         :action => lambda { sleep(1); $tasks_executed += 1 })
         @task2 = Task.new(:name => 'task2',
                          :scheduler => scheduler,
                          :schedule => schedule.clone,
-                         :action => lambda { sleep(1) })
+                         :action => lambda { sleep(1); $tasks_executed += 1 })
 
         scheduler.tasks.add_or_update(@task1)
         scheduler.tasks.add_or_update(@task2)
+        $tasks_executed = 0
       end
 
       describe "when there is no concurrent task limit set" do
@@ -241,10 +242,7 @@ module Recurrent
 
           [@task1, @task2].each {|task| task.thread.join }
 
-          finished_at = Time.now
-          elapsed = finished_at - current_time
-
-          elapsed.round.should == 1.seconds
+          $tasks_executed.should == 2
         end
       end
 
@@ -253,18 +251,16 @@ module Recurrent
           Configuration.maximum_concurrent_tasks = 1
         end
 
-        it "should run only up to the number of tasks specified at once" do
+        it "throw away tasks if there are too many tasks running" do
           current_time = Time.now
+
           [@task1, @task2].each do |task|
             task.execute(current_time)
           end
 
-          [@task1, @task2].each {|task| task.thread.join }
-
-          finished_at = Time.now
-          elapsed = finished_at - current_time
+          [@task1, @task2].each {|task| task.thread.try(:join) }
 
-          elapsed.round.should == 2.seconds
+          $tasks_executed.should == 1
         end
 
         after(:each) do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: recurrent
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 15
   prerelease: 
   segments: 
   - 0
-  - 3
+  - 4
   - 0
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors: 
 - Adam Kittelson
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-28 00:00:00 Z
+date: 2012-06-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: ice_cube
@@ -158,6 +158,7 @@ files:
 - .autotest
 - .gitignore
 - .rspec
+- .rvmrc
 - .zenflow
 - CHANGELOG.md
 - Gemfile
@@ -211,7 +212,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Task scheduler that doesn't need to bootstrap your Rails environment every time it executes a task the way running a rake task via cron does.