cloud-crowd 0.2.6 → 0.2.7

data/cloud-crowd.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.2.6'         # Keep version in sync with cloud-cloud.rb
-  s.date      = '2009-10-05'
+  s.version   = '0.2.7'         # Keep version in sync with cloud-cloud.rb
+  s.date      = '2009-10-19'
 
   s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
   s.summary     = "Parallel Processing for the Rest of Us"

data/config/database.example.yml

@@ -13,4 +13,4 @@
 # will do nicely:
 #
 # :adapter:  sqlite3
-# :database: cloud_crowd.db
+# :database: cloud_crowd.db

data/lib/cloud-crowd.rb

@@ -44,7 +44,7 @@ module CloudCrowd
   autoload :WorkUnit,     'cloud_crowd/models'
   
   # Keep this version in sync with the gemspec.
-  VERSION        = '0.2.6'
+  VERSION        = '0.2.7'
   
   # Increment the schema version when there's a backwards incompatible change.
   SCHEMA_VERSION = 3

data/lib/cloud_crowd/command_line.rb

@@ -24,6 +24,7 @@ Commands:
   node          Start up a worker node (only one node per machine, please)
   console       Launch a CloudCrowd console, connected to the central database
   load_schema   Load the schema into the database specified by database.yml
+  cleanup       Removes jobs that were finished over --days (7 by default) ago
   
   server -d [start | stop | restart]    Servers and nodes can be launched as
   node -d [start | stop | restart]      daemons, then stopped or restarted.
@@ -42,6 +43,7 @@ Options:
       when 'node'         then run_node(subcommand)
       when 'load_schema'  then run_load_schema
       when 'install'      then run_install(subcommand)
+      when 'cleanup'      then run_cleanup
       else                     usage
       end
     end
@@ -53,13 +55,14 @@ Options:
       require 'irb/completion'
       require 'pp'
       load_code
-      connect_to_database(true)
+      connect_to_database true
+      CloudCrowd::Server # Preload server to autoload classes.
+      Object.send(:include, CloudCrowd)
       IRB.start
     end
     
     # `crowd server` can either 'start', 'stop', or 'restart'.
     def run_server(subcommand)
-      ensure_config
       load_code
       subcommand ||= 'start'
       case subcommand
@@ -99,7 +102,7 @@ Options:
     # `crowd node` can either 'start', 'stop', or 'restart'.
     def run_node(subcommand)
       load_code
-      ENV['RACK_ENV'] = @options['environment']
+      ENV['RACK_ENV'] = @options[:environment]
       case (subcommand || 'start')
       when 'start'    then start_node
       when 'stop'     then stop_node
@@ -146,6 +149,13 @@ Options:
       install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
     end
     
+    # Clean up all Jobs in the CloudCrowd database older than --days old.
+    def run_cleanup
+      load_code
+      connect_to_database(true)
+      Job.cleanup_all(:days => @options[:days])
+    end
+    
     # Print `crowd` usage.
     def usage
       puts "\n#{@option_parser}\n"
@@ -163,6 +173,7 @@ Options:
     end
     
     # Parse all options for all commands.
+    # Valid options are: --config --port --environment --daemonize --days.
     def parse_options
       @options = {
         :environment  => 'production',
@@ -176,12 +187,15 @@ Options:
         opts.on('-p', '--port PORT', 'port number for server (central or node)') do |port_num|
           @options[:port] = port_num
         end
-        opts.on('-e', '--environment ENV', 'server environment (sinatra)') do |env|
+        opts.on('-e', '--environment ENV', 'server environment (defaults to production)') do |env|
           @options[:environment] = env
         end
         opts.on('-d', '--daemonize', 'run as a background daemon') do |daemonize|
           @options[:daemonize] = daemonize
         end
+        opts.on('--days NUM_DAYS', 'grace period before cleanup (7 by default)') do |days|
+          @options[:days] = days.to_i if days.match(/\A\d+\Z/)
+        end
         opts.on_tail('-v', '--version', 'show version') do
           require "#{CC_ROOT}/lib/cloud-crowd"
           puts "CloudCrowd version #{VERSION}"

data/lib/cloud_crowd/models/job.rb

@@ -7,6 +7,8 @@ module CloudCrowd
   class Job < ActiveRecord::Base
     include ModelStatus
     
+    CLEANUP_GRACE_PERIOD = 7 # That's a week.
+    
     has_many :work_units, :dependent => :destroy
     
     validates_presence_of :status, :inputs, :action, :options
@@ -14,6 +16,9 @@ module CloudCrowd
     before_validation_on_create :set_initial_status
     after_create                :queue_for_workers
     before_destroy              :cleanup_assets
+    
+    # Jobs that were last updated more than N days ago.
+    named_scope :older_than, lambda {|num| {:conditions => ['updated_at < ?', num.days.ago]} }
       
     # Create a Job from an incoming JSON request, and add it to the queue.
     def self.create_from_request(h)
@@ -26,6 +31,14 @@ module CloudCrowd
       )
     end
     
+    # Clean up all jobs beyond a certain age.
+    def self.cleanup_all(opts = {})
+      days = opts[:days] || CLEANUP_GRACE_PERIOD
+      self.complete.older_than(days).find_in_batches(:batch_size => 100) do |jobs|
+        jobs.each {|job| job.destroy }
+      end
+    end
+    
     # After work units are marked successful, we check to see if all of them have
     # finished, if so, continue on to the next phase of the job. 
     def check_for_completion

data/lib/cloud_crowd/models/node_record.rb

@@ -42,7 +42,7 @@ module CloudCrowd
     rescue RestClient::RequestFailed => e
       raise e unless e.http_code == 503 && e.http_body == Node::OVERLOADED_MESSAGE
       update_attribute(:busy, true) && false
-    rescue RestClient::Exception, Errno::ECONNREFUSED 
+    rescue RestClient::Exception, Errno::ECONNREFUSED, Timeout::Error
       # Couldn't post to node, assume it's gone away.
       destroy && false
     end

data/test/acceptance/test_failing_work_units.rb

@@ -14,7 +14,7 @@ class FailingWorkUnitsTest < Test::Unit::TestCase
     }.to_json
     assert browser.last_response.ok? 
     
-    job = CloudCrowd::Job.last
+    job = Job.last
     (CloudCrowd.config[:work_unit_retries] - 1).times do
       job.work_units.each {|unit| unit.fail('failed', 10) }
     end

data/test/acceptance/test_server.rb

@@ -11,8 +11,8 @@ class ServerTest < Test::Unit::TestCase
   context "The CloudCrowd::Server (Sinatra)" do
         
     setup do
-      CloudCrowd::Job.destroy_all
-      2.times { CloudCrowd::Job.make }
+      Job.destroy_all
+      2.times { Job.make }
     end
     
     should "be able to render the Operations Center (GET /)" do
@@ -52,20 +52,20 @@ class ServerTest < Test::Unit::TestCase
       job_info = JSON.parse(last_response.body)
       assert job_info['percent_complete'] == 0
       assert job_info['work_units'] == 1
-      assert CloudCrowd::Job.last.id == job_info['id']
+      assert Job.last.id == job_info['id']
     end
     
     should "be able to check in on the status of a job" do
-      get("/jobs/#{CloudCrowd::Job.last.id}")
+      get("/jobs/#{Job.last.id}")
       assert last_response.ok?
       assert JSON.parse(last_response.body)['percent_complete'] == 0
     end
     
     should "be able to clean up a job when we're done with it" do
-      id = CloudCrowd::Job.last.id
+      id = Job.last.id
       delete("/jobs/#{id}")
       assert last_response.successful? && last_response.empty?
-      assert !CloudCrowd::Job.find_by_id(id)
+      assert !Job.find_by_id(id)
     end
   
   end

data/test/unit/test_job.rb

@@ -5,7 +5,7 @@ class JobTest < Test::Unit::TestCase
   context "A CloudCrowd Job" do
         
     setup do
-      @job = CloudCrowd::Job.make
+      @job = Job.make
       @unit = @job.work_units.first
     end
     
@@ -25,7 +25,7 @@ class JobTest < Test::Unit::TestCase
     
     should "know its completion status" do
       assert !@job.all_work_units_complete?
-      @unit.update_attributes(:status => CloudCrowd::SUCCEEDED, :output => '{"output":"hello"}')
+      @unit.update_attributes(:status => SUCCEEDED, :output => '{"output":"hello"}')
       @job.check_for_completion
       assert @job.reload.all_work_units_complete?
       assert @job.percent_complete == 100
@@ -38,7 +38,7 @@ class JobTest < Test::Unit::TestCase
     end
     
     should "be able to create a job from a JSON request" do
-      job = CloudCrowd::Job.create_from_request(JSON.parse(<<-EOS
+      job = Job.create_from_request(JSON.parse(<<-EOS
       { "inputs"       : ["one", "two", "three"],
         "action"       : "graphics_magick",
         "email"        : "bob@example.com",
@@ -60,26 +60,38 @@ class JobTest < Test::Unit::TestCase
     end
     
     should "create jobs with a SPLITTING status for actions that have a split method defined" do
-      job = CloudCrowd::Job.create_from_request({'inputs' => ['1'], 'action' => 'process_pdfs'})
+      job = Job.create_from_request({'inputs' => ['1'], 'action' => 'process_pdfs'})
       assert job.splittable?
       assert job.splitting?
     end
     
     should "fire a callback when a job has finished, successfully or not" do
       @job.update_attribute(:callback_url, 'http://example.com/callback')
-      CloudCrowd::Job.any_instance.stubs(:fire_callback).returns(true)
-      CloudCrowd::Job.any_instance.expects(:fire_callback)
+      Job.any_instance.stubs(:fire_callback).returns(true)
+      Job.any_instance.expects(:fire_callback)
       @job.work_units.first.finish('{"output":"output"}', 10)
       assert @job.all_work_units_complete?
     end
     
     should "have a 'pretty' display of the Job's status" do
       assert @job.display_status == 'processing'
-      @job.update_attribute(:status, CloudCrowd::FAILED)
+      @job.update_attribute(:status, FAILED)
       assert @job.display_status == 'failed'
-      @job.update_attribute(:status, CloudCrowd::MERGING)
+      @job.update_attribute(:status, MERGING)
       assert @job.display_status == 'merging'
     end
+    
+    should "be able to clean up jobs that have aged beyond their use" do
+      count = Job.count
+      Job.cleanup_all
+      assert count == Job.count
+      Job.record_timestamps = false
+      @job.update_attributes :status => SUCCEEDED, :updated_at => 10.days.ago
+      Job.record_timestamps = true
+      Job.cleanup_all
+      assert count > Job.count
+      assert !Job.find_by_id(@job.id)
+    end
             
   end
   

data/test/unit/test_work_unit.rb

@@ -17,14 +17,14 @@ class WorkUnitTest < Test::Unit::TestCase
     
     should "know if its done" do
       assert !@unit.complete?
-      @unit.status = CloudCrowd::SUCCEEDED
+      @unit.status = SUCCEEDED
       assert @unit.complete?
-      @unit.status = CloudCrowd::FAILED
+      @unit.status = FAILED
       assert @unit.complete?
     end
     
     should "have JSON that includes job attributes" do
-      job = CloudCrowd::Job.make
+      job = Job.make
       unit_data = JSON.parse(job.work_units.first.to_json)
       assert unit_data['job_id'] == job.id
       assert unit_data['action'] == job.action

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: cloud-crowd
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors: 
 - Jeremy Ashkenas
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-05 00:00:00 -04:00
+date: 2009-10-19 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency