middle_management 0.0.1 → 1.0.0

data/README

@@ -50,6 +50,11 @@ Next 10 emails sent                                               10
 Next 10 emails sent               Stop 5 workers                  5
 Last 5 emails sent                Stop 5 workers                  0
 
+Accuracy and Caching Notes:
+* Job counts are queried no more than once per minute. If you create and destroy jobs through ActiveRecord, the internal counts will always be accurate; however if you delete jobs from a SQL tool, the counts will be out of date until the next count query is made.
+
+* To prevent flooding Heroku with API calls when large batches of jobs are created, worker API calls are made no more than once every 10 seconds. If a call is algorithmically ready to be made before that, it will be delayed 10 seconds. In a worst case scenario, this could allow your workers to run 10 seconds longer than needed.
+
 Testing in your heroku environment:
 
 Middle Management comes with a Slacker class, whose sole purpose is to create jobs that take some time to run. You can use this class to create jobs in your production environment and monitor how your workers scale accordingly.

data/lib/middle_management/job_modifications.rb

@@ -1,12 +1,20 @@
+require 'active_record'
 require 'delayed_job'
 require 'delayed/backend/active_record'
 
-class Delayed::Job < ::ActiveRecord::Base
-  after_create :enforce
-  after_destroy :enforce
-  
-  private
-  def self.enforce
-    MiddleManagement::Manager.enforce_number_of_current_jobs(Delayed::Job.where("run_at <= ? AND failed_at IS NULL AND locked_by IS NULL", Delayed::Backend::ActiveRecord::Job.db_time_now).count)
+module Delayed
+  module Backend
+    module ActiveRecord
+      class Job < ::ActiveRecord::Base
+        after_create do
+          MiddleManagement::Manager.track_creation
+          MiddleManagement::Manager.enforce_number_of_current_jobs
+        end
+        after_destroy do
+          MiddleManagement::Manager.track_completion
+          MiddleManagement::Manager.enforce_number_of_current_jobs
+        end
+      end
+    end
   end
-end
+end

data/lib/middle_management/manager.rb

@@ -1,13 +1,56 @@
 require 'heroku'
+require 'delayed_job'
+require 'delayed/backend/active_record'
 
 module MiddleManagement
   class Manager
-    def self.enforce_number_of_current_jobs(num)
-      self.set_num_workers(self.calculate_needed_worker_count(num)) if self.num_jobs_changes_worker_count?(num)
+    def self.enforce_number_of_current_jobs(_num_workers_last_set_at = nil, _current_worker_count = nil, _last_enforcement_job_set_for = nil)
+      # Do nothing if our worker figures are up-to-date
+      return if !self.num_jobs_changes_worker_count?(self.current_jobs_count)
+      
+      # Load cached values if they're supplied. This is to prevent different instances
+      # from re-querying the heroku API.
+      self.num_workers_last_set_at = _num_workers_last_set_at unless _num_workers_last_set_at.nil?
+      self.current_worker_count = _current_worker_count unless _current_worker_count.nil?
+      self.last_enforcement_job_set_for = _last_enforcement_job_set_for unless _last_enforcement_job_set_for.nil?
+      
+      # Set number of workers if we haven't set it recently
+      if self.num_workers_last_set_at.nil? || self.num_workers_last_set_at < 10.seconds.ago || self.current_worker_count.nil? || self.current_worker_count == 0
+        self.set_num_workers(self.calculate_needed_worker_count(self.current_jobs_count))
+        self.num_workers_last_set_at = Time.now
+      else
+        # Schedule job with cached data if we don't have one scheduled
+        if self.last_enforcement_job_set_for.nil? || Time.now > self.last_enforcement_job_set_for # Prevent multiple jobs within our window
+          self.last_enforcement_job_set_for = Time.now + 10.seconds
+          self.delay(:run_at => self.last_enforcement_job_set_for).enforce_number_of_current_jobs(self.num_workers_last_set_at, self.current_worker_count, self.last_enforcement_job_set_for)
+          # Make sure we have at least one worker running to cover the job.
+          self.set_num_workers(1) if self.current_worker_count == 0
+        end
+      end
+    end
+    
+    def self.track_creation
+      self.current_jobs_count += 1
+    end
+    
+    def self.track_completion
+      self.current_jobs_count -= 1
     end
     
     private
+    cattr_accessor :num_workers_last_set_at
+    cattr_accessor :last_enforcement_job_set_for
     cattr_accessor :current_worker_count
+    cattr_writer :current_jobs_count
+    cattr_accessor :current_jobs_count_last_queried
+    
+    def self.current_jobs_count
+      if @@current_jobs_count.nil? || self.current_jobs_count_last_queried.nil? || self.current_jobs_count_last_queried < 1.minute.ago
+        self.current_jobs_count = Delayed::Backend::ActiveRecord::Job.where("run_at <= ? AND failed_at IS NULL AND locked_by IS NULL", Delayed::Backend::ActiveRecord::Job.db_time_now).count
+        self.current_jobs_count_last_queried = Time.now
+      end
+      @@current_jobs_count
+    end
     
     def self.calculate_needed_worker_count(num_jobs)
       ideal_worker_count = num_jobs / MiddleManagement::Config::JOBS_PER_WORKER + 1
@@ -17,8 +60,8 @@ module MiddleManagement
     
     def self.num_jobs_changes_worker_count?(num_jobs)
       return false if num_jobs.nil?
-      return false if self.calculate_needed_worker_count(num_jobs) == current_worker_count
-      return true if current_worker_count.nil?
+      return false if self.calculate_needed_worker_count(num_jobs) == self.current_worker_count
+      return true if self.current_worker_count.nil?
       # Next two lines are verified in calculate_needed_worker_count(), but let's be safe since we're dealing with real money...
       return false if self.calculate_needed_worker_count(num_jobs) < MiddleManagement::Config::MIN_WORKERS
       return false if self.calculate_needed_worker_count(num_jobs) > MiddleManagement::Config::MAX_WORKERS
@@ -31,7 +74,7 @@ module MiddleManagement
     
     def self.set_num_workers(num_workers)
       self.get_heroku_client.set_workers(MiddleManagement::Config::HEROKU_APP, num_workers)
-      current_worker_count = num_workers
+      self.current_worker_count = num_workers
     end
   end
 end

data/lib/middle_management/version.rb

@@ -1,3 +1,3 @@
 module MiddleManagement
-  VERSION = "0.0.1"
+  VERSION = "1.0.0"
 end

data/spec/lib/middle_management/manager_spec.rb

@@ -1,6 +1,24 @@
 require File.expand_path("#{File.dirname(__FILE__)}/../../spec_helper")
 
 describe MiddleManagement::Manager do
+  describe "#track_creation" do
+    it "increments jobs count" do
+      MiddleManagement::Manager.should_receive(:current_jobs_count_last_queried).any_number_of_times.and_return(1.second.ago)
+      MiddleManagement::Manager.send(:current_jobs_count=, 3)
+      MiddleManagement::Manager.track_creation
+      MiddleManagement::Manager.send(:current_jobs_count).should == 4
+    end
+  end
+  
+  describe "#track_completion" do
+    it "decrements jobs count" do
+      MiddleManagement::Manager.should_receive(:current_jobs_count_last_queried).any_number_of_times.and_return(1.second.ago)
+      MiddleManagement::Manager.send(:current_jobs_count=, 3)
+      MiddleManagement::Manager.track_completion
+      MiddleManagement::Manager.send(:current_jobs_count).should == 2
+    end
+  end
+  
   describe "#enforce_number_of_current_jobs" do
     before do
       stub_config(:HEROKU_APP, "test_app")
@@ -9,23 +27,77 @@ describe MiddleManagement::Manager do
       @client_mock = mock("Heroku Client")
       MiddleManagement::Manager.should_receive(:get_heroku_client).any_number_of_times.and_return(@client_mock)
     end
+    describe "second call within 10 seconds" do
+      it "delays run for 10 seconds" do
+        MiddleManagement::Manager.send(:num_workers_last_set_at=, 5.seconds.ago)
+        MiddleManagement::Manager.send(:current_worker_count=, 5)
+        @client_mock.should_not_receive(:set_workers)
+        MiddleManagement::Manager.should_receive(:current_jobs_count).any_number_of_times.and_return(6)
+        delay_result = mock("Delay Object Result")
+        delay_result.should_receive(:enforce_number_of_current_jobs).exactly(:once)
+        MiddleManagement::Manager.should_receive(:delay).exactly(:once).and_return(delay_result)
+        MiddleManagement::Manager.enforce_number_of_current_jobs
+      end
+    end
+    describe "third call within 10 seconds" do
+      it "does not create job" do
+        MiddleManagement::Manager.send(:num_workers_last_set_at=, 5.seconds.ago)
+        MiddleManagement::Manager.send(:current_worker_count=, 5)
+        @client_mock.should_not_receive(:set_workers)
+        MiddleManagement::Manager.should_receive(:current_jobs_count).any_number_of_times.and_return(6)
+        MiddleManagement::Manager.should_not_receive(:delay)
+        MiddleManagement::Manager.send(:last_enforcement_job_set_for=, 5.seconds.from_now)
+        MiddleManagement::Manager.enforce_number_of_current_jobs
+      end
+    end
     describe "changes number of workers" do
       it "makes api call" do
+        MiddleManagement::Manager.send(:num_workers_last_set_at=, nil)
         MiddleManagement::Manager.send(:current_worker_count=, 5)
         @client_mock.should_receive(:set_workers).exactly(:once)
-        MiddleManagement::Manager.enforce_number_of_current_jobs(6)
+        MiddleManagement::Manager.should_receive(:current_jobs_count).any_number_of_times.and_return(6)
+        MiddleManagement::Manager.enforce_number_of_current_jobs
       end
     end
     describe "no change to number of workers" do
       it "does not make api call" do
+        MiddleManagement::Manager.send(:num_workers_last_set_at=, nil)
         MiddleManagement::Manager.send(:current_worker_count=, 5)
+        MiddleManagement::Manager.should_receive(:current_jobs_count).any_number_of_times.and_return(5)
         @client_mock.should_not_receive(:set_workers)
-        MiddleManagement::Manager.enforce_number_of_current_jobs(5)
+        MiddleManagement::Manager.enforce_number_of_current_jobs
       end
     end
   end
   
   describe "private methods" do
+    describe "#current_jobs_count" do
+      it "queries if hasn't queried" do
+        MiddleManagement::Manager.send(:current_jobs_count=, nil)
+        MiddleManagement::Manager.send(:current_jobs_count_last_queried=, nil)
+        MiddleManagement::Manager.should_receive(:current_jobs_count_last_queried).any_number_of_times.and_return(nil)
+        r = mock("Result")
+        r.should_receive(:count).exactly(:once).and_return(3)
+        Delayed::Backend::ActiveRecord::Job.should_receive(:where).exactly(:once).and_return(r)
+        MiddleManagement::Manager.send(:current_jobs_count).should == 3
+      end
+      it "queries if last query was > 1 minute ago" do
+        MiddleManagement::Manager.send(:current_jobs_count=, nil)
+        MiddleManagement::Manager.send(:current_jobs_count_last_queried=, 90.seconds.ago)
+        MiddleManagement::Manager.should_receive(:current_jobs_count_last_queried).any_number_of_times.and_return(nil)
+        r = mock("Result")
+        r.should_receive(:count).exactly(:once).and_return(3)
+        Delayed::Backend::ActiveRecord::Job.should_receive(:where).exactly(:once).and_return(r)
+        MiddleManagement::Manager.send(:current_jobs_count).should == 3
+      end
+      it "uses cache if last query was < 1 minute ago" do
+        MiddleManagement::Manager.send(:current_jobs_count=, 5)
+        MiddleManagement::Manager.send(:current_jobs_count_last_queried=, 59.seconds.ago)
+        Delayed::Backend::ActiveRecord::Job.should_not_receive(:where)
+        MiddleManagement::Manager.send(:current_jobs_count).should == 5
+      end
+    end
+    
     describe "#calculate_needed_worker_count" do
       describe "1 job per worker" do
         before do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: middle_management
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 23
   prerelease: false
   segments: 
+  - 1
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors: 
 - Robby Grossman
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-16 00:00:00 -05:00
+date: 2011-01-20 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -135,7 +135,6 @@ files:
 - lib/middle_management/slacker.rb
 - lib/middle_management/version.rb
 - middle_management.gemspec
-- spec/lib/middle_management/job_modifications_spec.rb
 - spec/lib/middle_management/manager_spec.rb
 - spec/lib/middle_management_spec.rb
 - spec/spec_helper.rb
@@ -174,7 +173,6 @@ signing_key:
 specification_version: 3
 summary: Delayed Job worker management for Heroku.
 test_files: 
-- spec/lib/middle_management/job_modifications_spec.rb
 - spec/lib/middle_management/manager_spec.rb
 - spec/lib/middle_management_spec.rb
 - spec/spec_helper.rb

data/spec/lib/middle_management/job_modifications_spec.rb

@@ -1,18 +0,0 @@
-require File.expand_path("#{File.dirname(__FILE__)}/../../spec_helper")
-
-describe Delayed::Job do
-  before do
-    @client_mock = mock("Heroku Client")
-    MiddleManagement::Manager.should_receive(:get_heroku_client).any_number_of_times.and_return(@client_mock)
-  end
-  
-  describe "#enforce" do
-    it "micromanages remaining jobs" do
-      MiddleManagement::Manager.should_receive(:enforce_number_of_current_jobs).with(3).exactly(:once)
-      Delayed::Job.should_receive(:count).exactly(:once).and_return(3)
-      Delayed::Backend::ActiveRecord::Job.should_receive(:db_time_now).and_return(Time.now)
-      Delayed::Job.should_receive(:where).any_number_of_times.and_return(Delayed::Job)
-      Delayed::Job.send(:enforce)
-    end
-  end
-end