recurrent 0.2.0 → 0.3.0

data/.gitignore

@@ -5,3 +5,5 @@ Gemfile.lock
 .bundle
 *.gem
 pkg/*
+/.idea
+.zenflow-log

data/.zenflow

@@ -0,0 +1,10 @@
+--- 
+confirm_review: true
+backup_remote: false
+release_branch: production
+project: recurrent
+qa_branch: false
+staging_branch: false
+development_branch: master
+remote: origin
+confirm_staging: false

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+--------------------------------------------------------------------------------
+                          ^ ADD NEW CHANGES ABOVE ^
+--------------------------------------------------------------------------------
+
+CHANGELOG
+=========
+

data/README.markdown

@@ -121,6 +121,15 @@ How long to wait before killing tasks that are still running.
 
      configure.wait_for_running_tasks_on_exit_for = 10.seconds
 
+###Limiting the number of concurrent tasks
+To limit the number of tasks that run simultaneously:
+
+    configure.maximum_concurrent_tasks = 5
+
+This will run up to the above number of tasks simultaneously, other tasks will wait until a slot opens up before executing. Tasks that run more frequently, e.g. once a minute, will have precedence over tasks that run once an hour or day etc.
+
+
+
 Submitting an Issue
 -------------------
 We use the [GitHub issue tracker](http://github.com/zencoder/recurrent/issues) to track bugs and

data/lib/recurrent.rb

@@ -10,5 +10,6 @@ require 'recurrent/configuration'
 require 'recurrent/logger'
 require 'recurrent/scheduler'
 require 'recurrent/task'
+require 'recurrent/task_collection'
 require 'recurrent/version'
 require 'recurrent/worker'

data/lib/recurrent/configuration.rb

@@ -3,7 +3,7 @@ module Recurrent
 
     class << self
 
-      attr_accessor :logging, :wait_for_running_tasks_on_exit_for
+      attr_accessor :logging, :wait_for_running_tasks_on_exit_for, :maximum_concurrent_tasks
 
       def self.block_accessor(*fields)
         fields.each do |field|

data/lib/recurrent/scheduler.rb

@@ -1,12 +1,14 @@
 module Recurrent
   class Scheduler
 
-    attr_accessor :tasks, :logger
+    attr_accessor :tasks, :logger, :executing_tasks, :mutex
 
     def initialize(task_file=nil)
-      @tasks = []
+      @tasks = TaskCollection.new
       identifier = "host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
       @logger = Logger.new(identifier)
+      @mutex = Mutex.new
+      @executing_tasks = 0
       eval(File.read(task_file)) if task_file
     end
 
@@ -87,30 +89,16 @@ module Recurrent
 
     def every(frequency, key, options={}, &block)
       logger.info "Adding Task: #{key}"
-      @tasks << Task.new(:name => key,
-                         :schedule => create_schedule(key, frequency, options[:start_time]),
-                         :action => block,
-                         :save => options[:save],
-                         :logger => logger)
+      task = Task.new(:name => key,
+                      :schedule => create_schedule(key, frequency, options[:start_time]),
+                      :action => block,
+                      :save => options[:save],
+                      :logger => logger,
+                      :scheduler => self)
+      @tasks.add_or_update(task)
       logger.info "| #{key} added to Scheduler"
     end
 
-    def next_task_time
-      tasks.map { |task| task.next_occurrence }.sort.first
-    end
-
-    def running_tasks
-      tasks.select do |task|
-        task.running?
-      end
-    end
-
-    def tasks_at_time(time)
-      tasks.select do |task|
-        task.next_occurrence == time
-      end
-    end
-
     def use_saved_schedule_if_rules_match(saved_schedule, new_schedule)
       if new_schedule.has_same_rules? saved_schedule
         logger.info "| Schedule matches a saved schedule, using saved schedule."
@@ -121,6 +109,18 @@ module Recurrent
       end
     end
 
+    def increment_executing_tasks
+      mutex.synchronize do
+        @executing_tasks += 1
+      end
+    end
+
+    def decrement_executing_tasks
+      mutex.synchronize do
+       @executing_tasks -= 1
+      end
+    end
+
     def self.define_frequencies(*frequencies)
       frequencies.each do |frequency|
         method_name = frequency == :day ? :daily? : :"#{frequency}ly?"

data/lib/recurrent/task.rb

@@ -1,6 +1,8 @@
 module Recurrent
+  class TooManyExecutingTasks < StandardError; end
+
   class Task
-    attr_accessor :action, :name, :logger, :save, :schedule, :thread
+    attr_accessor :action, :name, :logger, :save, :schedule, :thread, :scheduler
 
     def initialize(options={})
       @name = options[:name]
@@ -8,6 +10,7 @@ module Recurrent
       @action = options[:action]
       @save = options[:save]
       @logger = options[:logger]
+      @scheduler = options[:scheduler]
       Configuration.save_task_schedule.call(name, schedule) if Configuration.save_task_schedule
     end
 
@@ -16,13 +19,15 @@ module Recurrent
       @thread = Thread.new do
         Thread.current["execution_time"] = execution_time
         begin
-          if Configuration.load_task_return_value && action.arity == 1
-            previous_value = Configuration.load_task_return_value.call(name)
-            return_value = action.call(previous_value)
+          if Configuration.maximum_concurrent_tasks.present?
+            limit_execution_to_max_concurrency
           else
-            return_value = action.call
+            call_action
           end
-          save_results(return_value) if save?
+        rescue TooManyExecutingTasks
+          scheduler.decrement_executing_tasks
+          sleep(0.1)
+          retry
         rescue => e
           logger.warn("#{name} - #{e.message}")
           logger.warn(e.backtrace)
@@ -30,10 +35,34 @@ module Recurrent
       end
     end
 
+    def limit_execution_to_max_concurrency
+      if (scheduler.increment_executing_tasks <= Configuration.maximum_concurrent_tasks) && task_is_next_in_line?
+        call_action
+        scheduler.decrement_executing_tasks
+      else
+        raise TooManyExecutingTasks
+      end
+    end
+
+    def task_is_next_in_line?
+      self == scheduler.tasks.next_for_execution_at_time(Thread.current["execution_time"])
+    end
+
+    def call_action
+      if Configuration.load_task_return_value && action.arity == 1
+        previous_value = Configuration.load_task_return_value.call(name)
+
+        return_value = action.call(previous_value)
+      else
+        return_value = action.call
+      end
+      save_results(return_value) if save?
+    end
+
     def handle_still_running(current_time)
       logger.info "#{name}: Execution from #{thread['execution_time'].to_s(:seconds)} still running, aborting this execution."
       if Configuration.handle_slow_task
-        Configuration.handle_slow_task.call(name, current_time)
+        Configuration.handle_slow_task.call(name, current_time, thread['execution_time'])
       end
     end
 

data/lib/recurrent/task_collection.rb

@@ -0,0 +1,72 @@
+module Recurrent
+  class TaskCollection
+
+    def initialize
+      @mutex = Mutex.new
+      @tasks = []
+    end
+
+    def add_or_update(new_task)
+      @mutex.synchronize do
+        old_task_index = @tasks.index {|task| task.name == new_task.name }
+        if old_task_index
+          @tasks[old_task_index].schedule = new_task.schedule
+          @tasks[old_task_index].action = new_task.action
+        else
+          @tasks << new_task
+        end
+      end
+    end
+
+    def next_for_execution_at_time(time)
+      @mutex.synchronize do
+        tasks_running_at_time = @tasks.select {|task| task.running? && task.thread['execution_time'] == time }
+        TaskCollection.sort_by_frequency(tasks_running_at_time).first
+      end
+    end
+
+    def next_execution_time
+      @mutex.synchronize do
+        @tasks.map { |task| task.next_occurrence }.sort.first
+      end
+    end
+
+    def remove(name)
+      @mutex.synchronize do
+        @tasks.reject! {|task| task.name == name }
+      end
+    end
+
+    def running
+      @mutex.synchronize do
+        @tasks.select {|task| task.running? }
+      end
+    end
+
+    def scheduled_to_execute_at(time, opts={})
+      @mutex.synchronize do
+        current_tasks = @tasks.select {|task| task.next_occurrence == time }
+        if opts[:sort_by_frequency]
+          TaskCollection.sort_by_frequency(current_tasks)
+        else
+          current_tasks
+        end
+      end
+    end
+
+    def self.sort_by_frequency(task_list)
+      task_list.sort_by do |task|
+        task.schedule.rrules.sort_by do |rule|
+          rule.frequency_in_seconds
+        end.first.frequency_in_seconds
+      end
+    end
+
+
+    def method_missing(id, *args, &block)
+      @mutex.synchronize do
+        @tasks.send(id, *args, &block)
+      end
+    end
+  end
+end

data/lib/recurrent/version.rb

@@ -1,3 +1,3 @@
 module Recurrent
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/recurrent/worker.rb

@@ -40,8 +40,8 @@ module Recurrent
 
     def execute
       loop do
-        execution_time = scheduler.next_task_time
-        tasks_to_execute = scheduler.tasks_at_time(execution_time)
+        execution_time = scheduler.tasks.next_execution_time
+        tasks_to_execute = scheduler.tasks.scheduled_to_execute_at(execution_time, :sort_by_frequency => !!Configuration.maximum_concurrent_tasks)
 
         wait_for_running_tasks && break if $exit
 
@@ -80,12 +80,12 @@ module Recurrent
     end
 
     def wait_for_running_tasks_for(seconds)
-      while scheduler.running_tasks.any? do
+      while scheduler.tasks.running.any? do
         logger.info "Killing running tasks in #{seconds.inspect}."
         seconds -= 1
         sleep(1)
         if seconds == 0
-          scheduler.running_tasks.each do |task|
+          scheduler.tasks.running.each do |task|
             logger.info "Killing #{task.name}."
             task.thread = nil unless task.thread.try(:kill).try(:alive?)
           end
@@ -95,7 +95,7 @@ module Recurrent
     end
 
     def wait_for_running_tasks_indefinitely
-      if task = scheduler.running_tasks.first
+      if task = scheduler.tasks.running.first
         logger.info "Waiting for #{task.name} to finish."
         task.thread.try(:join)
         wait_for_running_tasks_indefinitely

data/spec/scheduler_spec.rb

@@ -195,36 +195,17 @@ module Recurrent
     describe "#next_task_time" do
       context "when there are multiple tasks" do
         it "should return the soonest time at which a task is scheduled" do
-          task1 = stub('task1')
+          task1 = stub('task1', :name => :task1)
           task1.stub(:next_occurrence).and_return(10.minutes.from_now)
-          task2 = stub('task2')
+          task2 = stub('task2', :name => :task2)
           task2.stub(:next_occurrence).and_return(5.minutes.from_now)
-          task3 = stub('task3')
+          task3 = stub('task3', :name => :task3)
           task3.stub(:next_occurrence).and_return(15.minutes.from_now)
           schedule = Scheduler.new
-          schedule.tasks << task1
-          schedule.tasks << task2
-          schedule.tasks << task3
-          schedule.next_task_time.should == task2.next_occurrence
-        end
-      end
-    end
-
-    describe "#tasks_at_time" do
-      context "when there are multiple tasks" do
-        it "should return all the tasks whose next_occurrence is at the specified time" do
-          in_five_minutes = 5.minutes.from_now
-          task1 = stub('task1')
-          task1.stub(:next_occurrence).and_return(in_five_minutes)
-          task2 = stub('task2')
-          task2.stub(:next_occurrence).and_return(10.minutes.from_now)
-          task3 = stub('task3')
-          task3.stub(:next_occurrence).and_return(in_five_minutes)
-          schedule = Scheduler.new
-          schedule.tasks << task1
-          schedule.tasks << task2
-          schedule.tasks << task3
-          schedule.tasks_at_time(in_five_minutes).should =~ [task1, task3]
+          schedule.tasks.add_or_update(task1)
+          schedule.tasks.add_or_update(task2)
+          schedule.tasks.add_or_update(task3)
+          schedule.tasks.next_execution_time.should == task2.next_occurrence
         end
       end
     end

data/spec/task_collection_spec.rb

@@ -0,0 +1,172 @@
+require 'spec_helper'
+
+module Recurrent
+  describe TaskCollection do
+    before(:all) do
+      Configuration.logging = "quiet"
+    end
+
+    describe "#add_or_update_task" do
+      before(:each) do
+        @tasks = TaskCollection.new
+      end
+
+      context "when adding a new task" do
+        before(:each) do
+          @task = Task.new(:name => :new_task)
+        end
+
+        it "adds the task to the list of tasks" do
+          @tasks.size.should == 0
+          @tasks.add_or_update(@task)
+          @tasks.size.should == 1
+          @tasks.first.should == @task
+        end
+      end
+
+      context "when updating a task" do
+        before(:each) do
+          @original_frequency = Scheduler.new.create_schedule(:task, 5.seconds)
+          @original_action    = proc { "I am the original task!" }
+          @original_task      = Task.new(:name      => :task,
+                                         :frequency => @original_frequency,
+                                         :action    => @original_action)
+
+          @new_frequency = Scheduler.new.create_schedule(:task, 10.seconds)
+          @new_action    = proc { "I am the new task!" }
+          @new_task      = Task.new(:name      => :task,
+                                    :frequency => @new_frequency,
+                                    :action    => @new_action)
+          @tasks << @original_task
+        end
+
+        context "before updating the task" do
+          it "has one task" do
+            @tasks.size.should == 1
+          end
+
+          it "has the original task's action" do
+            @tasks.first.action.call.should == "I am the original task!"
+          end
+
+          it "has the original task's frequency" do
+            @tasks.first.schedule.should == @original_schedule
+          end
+        end
+
+        context "after updating the task" do
+          before(:each) do
+            @tasks.add_or_update(@new_task)
+          end
+
+          it "has one task" do
+            @tasks.size.should == 1
+          end
+
+          it "has the new task's action" do
+            @tasks.first.action.call.should == "I am the new task!"
+          end
+
+          it "has the new task's frequency" do
+            @tasks.first.schedule.should == @new_schedule
+          end
+        end
+      end
+    end
+
+    describe "#remove_task" do
+      context "A TaskCollection with 3 tasks" do
+        before(:each) do
+          @tasks     = TaskCollection.new
+          @task1     = Task.new(:name => :task1)
+          @task2     = Task.new(:name => :task2)
+          @task3     = Task.new(:name => :task3)
+          @tasks.add_or_update(@task1)
+          @tasks.add_or_update(@task2)
+          @tasks.add_or_update(@task3)
+        end
+
+        it "has 3 tasks" do
+          @tasks.size.should == 3
+          (@tasks | []).should == [@task1, @task2, @task3]
+        end
+
+        context "that removes a task" do
+          before(:each) do
+            @tasks.remove(:task2)
+          end
+
+          it "has 2 tasks" do
+            @tasks.size.should == 2
+            (@tasks | []).should == [@task1, @task3]
+          end
+        end
+      end
+    end
+
+    describe "#scheduled_to_execute_at" do
+      context "when there are multiple tasks" do
+        it "should return all the tasks whose next_occurrence is at the specified time" do
+          task_1_schedule = IceCube::Schedule.new(Time.utc(2012, 1, 10))
+          task_1_schedule.add_recurrence_rule(IceCube::Rule.minutely(10))
+
+          task_2_schedule = IceCube::Schedule.new(Time.utc(2012, 1, 10))
+          task_2_schedule.add_recurrence_rule(IceCube::Rule.minutely(5))
+
+          task_3_schedule = IceCube::Schedule.new(Time.utc(2012, 1, 10))
+          task_3_schedule.add_recurrence_rule(IceCube::Rule.minutely(1))
+
+          current_time = Time.utc(2012, 1, 10, 14, 4)
+          Timecop.freeze(current_time)
+
+          task1     = Task.new(:name     => 'task1',
+                               :schedule => task_1_schedule)
+          task2     = Task.new(:name     => 'task2',
+                               :schedule => task_2_schedule)
+          task3     = Task.new(:name     => 'task3',
+                               :schedule => task_3_schedule)
+          tasks = TaskCollection.new
+          tasks.add_or_update(task1)
+          tasks.add_or_update(task2)
+          tasks.add_or_update(task3)
+
+          tasks.scheduled_to_execute_at(Time.utc(2012, 1, 10, 14, 5)).should =~ [task2, task3]
+          Timecop.return
+        end
+
+        context "when :sort_by_frequency => true is passed as an option" do
+          it "should return the sorted by frequency, most frequent first" do
+            task_1_schedule = IceCube::Schedule.new(Time.utc(2012, 1, 10))
+            task_1_schedule.add_recurrence_rule(IceCube::Rule.minutely(10))
+
+            task_2_schedule = IceCube::Schedule.new(Time.utc(2012, 1, 10))
+            task_2_schedule.add_recurrence_rule(IceCube::Rule.minutely(5))
+
+            task_3_schedule = IceCube::Schedule.new(Time.utc(2012, 1, 10))
+            task_3_schedule.add_recurrence_rule(IceCube::Rule.minutely(1))
+
+            current_time = Time.utc(2012, 1, 10, 14, 4)
+            Timecop.freeze(current_time)
+
+            task1     = Task.new(:name     => 'task1',
+                                 :schedule => task_1_schedule)
+            task2     = Task.new(:name     => 'task2',
+                                 :schedule => task_2_schedule)
+            task3     = Task.new(:name     => 'task3',
+                                 :schedule => task_3_schedule)
+            tasks = TaskCollection.new
+            tasks.add_or_update(task1)
+            tasks.add_or_update(task2)
+            tasks.add_or_update(task3)
+
+            first_task, second_task = *tasks.scheduled_to_execute_at(Time.utc(2012, 1, 10, 14, 5), :sort_by_frequency => true)
+            first_task.should == task3
+            second_task.should == task2
+            Timecop.return
+          end
+        end
+      end
+    end
+  end
+end
+

data/spec/task_spec.rb

@@ -72,13 +72,6 @@ module Recurrent
           Configuration.load_task_return_value = nil
         end
       end
-
-      context "load_task_return_value is not configured" do
-
-      end
-
-
-
     end
 
     describe "#next_occurrence" do
@@ -129,7 +122,7 @@ module Recurrent
 
         it "logs that the task is still running and calls the method" do
           @task.logger.should_receive(:info).with("handle_still_running_test: Execution from #{@executing_task_time.to_s(:seconds)} still running, aborting this execution.")
-          Configuration.handle_slow_task.should_receive(:call).with('handle_still_running_test', @current_time)
+          Configuration.handle_slow_task.should_receive(:call).with('handle_still_running_test', @current_time, @executing_task_time)
           @task.handle_still_running(@current_time)
         end
 
@@ -219,7 +212,66 @@ module Recurrent
           t.running?.should be_false
         end
       end
+    end
 
+    describe "Restricting to a maximum number of concurrent tasks" do
+      before(:each) do
+        scheduler = Scheduler.new
+        schedule = IceCube::Schedule.new(Time.now.utc.beginning_of_day)
+        schedule.add_recurrence_rule IceCube::Rule.minutely(1)
+        @task1 = Task.new(:name => 'task1',
+                         :scheduler => scheduler,
+                         :schedule => schedule.clone,
+                         :action => lambda { sleep(1)})
+        @task2 = Task.new(:name => 'task2',
+                         :scheduler => scheduler,
+                         :schedule => schedule.clone,
+                         :action => lambda { sleep(1) })
+
+        scheduler.tasks.add_or_update(@task1)
+        scheduler.tasks.add_or_update(@task2)
+      end
+
+      describe "when there is no concurrent task limit set" do
+        it "should run all tasks at the same time" do
+          current_time = Time.now
+          [@task1, @task2].each do |task|
+            task.execute(current_time)
+          end
+
+          [@task1, @task2].each {|task| task.thread.join }
+
+          finished_at = Time.now
+          elapsed = finished_at - current_time
+
+          elapsed.round.should == 1.seconds
+        end
+      end
+
+      describe "when there is a maximum concurrency limit set" do
+        before(:each) do
+          Configuration.maximum_concurrent_tasks = 1
+        end
+
+        it "should run only up to the number of tasks specified at once" do
+          current_time = Time.now
+          [@task1, @task2].each do |task|
+            task.execute(current_time)
+          end
+
+          [@task1, @task2].each {|task| task.thread.join }
+
+          finished_at = Time.now
+          elapsed = finished_at - current_time
+
+          elapsed.round.should == 2.seconds
+        end
+
+        after(:each) do
+          Configuration.maximum_concurrent_tasks = nil
+        end
+      end
     end
+
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: recurrent
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: 
   segments: 
   - 0
-  - 2
+  - 3
   - 0
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Adam Kittelson
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-13 00:00:00 Z
+date: 2012-01-28 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: ice_cube
@@ -158,6 +158,8 @@ files:
 - .autotest
 - .gitignore
 - .rspec
+- .zenflow
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.markdown
@@ -170,12 +172,14 @@ files:
 - lib/recurrent/logger.rb
 - lib/recurrent/scheduler.rb
 - lib/recurrent/task.rb
+- lib/recurrent/task_collection.rb
 - lib/recurrent/version.rb
 - lib/recurrent/worker.rb
 - recurrent.gemspec
 - spec/logger_spec.rb
 - spec/scheduler_spec.rb
 - spec/spec_helper.rb
+- spec/task_collection_spec.rb
 - spec/task_spec.rb
 - spec/worker_spec.rb
 homepage: http://github.com/zencoder/recurrent
@@ -215,5 +219,6 @@ test_files:
 - spec/logger_spec.rb
 - spec/scheduler_spec.rb
 - spec/spec_helper.rb
+- spec/task_collection_spec.rb
 - spec/task_spec.rb
 - spec/worker_spec.rb