cloud-crowd 0.1.0 → 0.2.0

data/test/acceptance/test_failing_work_units.rb

@@ -4,7 +4,7 @@ require 'test_helper'
 class FailingWorkUnitsTest < Test::Unit::TestCase
 
   should "retry work units when they fail" do
-    browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::App))
+    browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
     
     browser.post '/jobs', :job => {
       'action'  => 'failure_testing',

data/test/acceptance/{test_app.rb → test_server.rb}

@@ -1,14 +1,14 @@
 require 'test_helper'
 
-class AppTest < Test::Unit::TestCase
+class ServerTest < Test::Unit::TestCase
   
   include Rack::Test::Methods
   
   def app
-    CloudCrowd::App
+    CloudCrowd::Server
   end
   
-  context "The CloudCrowd::App (Sinatra)" do
+  context "The CloudCrowd::Server (Sinatra)" do
         
     setup do
       CloudCrowd::Job.destroy_all
@@ -17,7 +17,7 @@ class AppTest < Test::Unit::TestCase
     
     should "be able to render the Operations Center (GET /)" do
       get '/'
-      assert last_response.body.include? '<div id="workers">'
+      assert last_response.body.include? '<div id="nodes">'
       assert last_response.body.include? '<div id="graphs">'
     end
     
@@ -29,17 +29,17 @@ class AppTest < Test::Unit::TestCase
       assert resp['work_unit_count'] == 2
     end
     
-    should "be able to check in a worker daemon, and then check out a work unit" do
-      put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
-      assert last_response.successful? && last_response.empty?
-      post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
-      checked_out = JSON.parse(last_response.body)
-      assert checked_out['action'] == 'graphics_magick'
-      assert checked_out['attempts'] == 0
-      assert checked_out['status'] == CloudCrowd::PROCESSING
-      status_check = JSON.parse(get('/worker/101@localhost').body)
-      assert checked_out == status_check
-    end
+    # should "be able to check in a worker daemon, and then check out a work unit" do
+    #   put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
+    #   assert last_response.successful? && last_response.empty?
+    #   post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
+    #   checked_out = JSON.parse(last_response.body)
+    #   assert checked_out['action'] == 'graphics_magick'
+    #   assert checked_out['attempts'] == 0
+    #   assert checked_out['status'] == CloudCrowd::PROCESSING
+    #   status_check = JSON.parse(get('/worker/101@localhost').body)
+    #   assert checked_out == status_check
+    # end
     
     should "have a heartbeat" do
       assert get('/heartbeat').body == 'buh-bump'

data/test/acceptance/test_word_count.rb

@@ -6,7 +6,7 @@ class WordCountTest < Test::Unit::TestCase
     
     setup do
       @asset_store = AssetStore.new
-      @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::App))
+      @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
       @browser.put('/worker', :name => 'test_worker', :thread_status => 'sleeping')
       post_job_to_count_words_in_this_file
       @job_id = JSON.parse(@browser.last_response.body)['id']
@@ -24,15 +24,9 @@ class WordCountTest < Test::Unit::TestCase
     end
     
     should "be able to perform the processing stage of a word_count" do
-      @browser.post('/work', :worker_name => 'test_worker', :worker_actions => 'word_count')
-      assert @browser.last_response.ok?
-      info = JSON.parse(@browser.last_response.body)
-      assert info['status'] == 1
-      assert info['action'] == 'word_count'
-      assert info['input'] == "file://#{File.expand_path(__FILE__)}"
-      action = CloudCrowd.actions['word_count'].new(info['status'], info['input'], info['options'], @asset_store)
+      action = CloudCrowd.actions['word_count'].new(1, "file://#{File.expand_path(__FILE__)}", {}, @asset_store)
       count = action.process
-      assert count == 128
+      assert count == 104
     end
     
   end

data/test/blueprints.rb

@@ -11,7 +11,6 @@ end
 CloudCrowd::WorkUnit.blueprint do
   job               { CloudCrowd::Job.make }
   status            { CloudCrowd::PROCESSING }
-  worker_record_id  { nil }
   input             { Sham.url }
   action            { 'graphics_magick' }
 end

data/test/config/config.ru

@@ -13,5 +13,5 @@ CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
 CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
 
 map '/' do
-  run CloudCrowd::App
+  run CloudCrowd::Server
 end

data/test/config/config.yml

@@ -1,7 +1,5 @@
-:num_workers:             4
-:min_worker_wait:         1
-:max_worker_wait:         20
+:max_workers:             10
 :work_unit_retries:       3
 
 :central_server:          http://localhost:9173
-:storage:                 filesystem
+:storage:                 filesystem

data/test/unit/test_action.rb

@@ -27,7 +27,7 @@ class ActionTest < Test::Unit::TestCase
     end
     
     should "be able to save (to the filesystem while testing)" do
-      assert @action.save(@action.input_path) == "file://#{CloudCrowd::AssetStore::LOCAL_STORAGE_PATH}/word_count/job_1/unit_1/test_action.rb"
+      assert @action.save(@action.input_path) == "file://#{@store.local_storage_path}/word_count/job_1/unit_1/test_action.rb"
     end
     
     should "be able to clean up after itself" do

data/test/unit/test_configuration.rb

@@ -5,7 +5,7 @@ class ConfigurationTest < Test::Unit::TestCase
   context "CloudCrowd Configuration" do
             
     should "have read in config.yml" do
-      assert CloudCrowd.config[:num_workers] == 4
+      assert CloudCrowd.config[:max_workers] == 10
       assert CloudCrowd.config[:storage] == 'filesystem'
     end
     

data/test/unit/test_job.rb

@@ -26,6 +26,7 @@ class JobTest < Test::Unit::TestCase
     should "know its completion status" do
       assert !@job.all_work_units_complete?
       @unit.update_attributes(:status => CloudCrowd::SUCCEEDED, :output => '{"output":"hello"}')
+      @job.check_for_completion
       assert @job.reload.all_work_units_complete?
       assert @job.percent_complete == 100
       assert @job.outputs == "[\"hello\"]"
@@ -60,6 +61,8 @@ class JobTest < Test::Unit::TestCase
     end
     
     should "fire a callback when a job has finished, successfully or not" do
+      @job.update_attribute(:callback_url, 'http://example.com/callback')
+      CloudCrowd::Job.any_instance.stubs(:fire_callback).returns(true)
       CloudCrowd::Job.any_instance.expects(:fire_callback)
       @job.work_units.first.finish('{"output":"output"}', 10)
       assert @job.all_work_units_complete?

data/test/unit/test_work_unit.rb

@@ -21,8 +21,6 @@ class WorkUnitTest < Test::Unit::TestCase
       assert @unit.complete?
       @unit.status = CloudCrowd::FAILED
       assert @unit.complete?
-      @unit.expects :check_for_job_completion
-      @unit.save
     end
     
     should "have JSON that includes job attributes" do
@@ -34,10 +32,10 @@ class WorkUnitTest < Test::Unit::TestCase
     end
     
     should "be able to retry, on failure" do
-      @unit.update_attribute :worker_record_id, 10
+      @unit.update_attribute :worker_pid, 7337
       assert @unit.attempts == 0
       @unit.fail('oops', 10)
-      assert @unit.worker_record == nil
+      assert @unit.worker_pid == nil
       assert @unit.attempts == 1
       assert @unit.processing?
       @unit.fail('oops again', 10)

data/views/{index.erb → operations_center.erb}

@@ -36,25 +36,30 @@
       <div id="sidebar_background"></div>
       <div id="sidebar_bottom" class="sidebar_back"></div>
       <div id="sidebar_header" class="small_caps">
-        <span class="has_workers">Active Worker Daemons</span>
-        <span class="no_workers">No Workers Running&hellip;</span>
+        <span class="has_nodes"></span>
+        <span class="no_nodes">No Nodes Online&hellip;</span>
       </div>
-      <div id="workers">
-        <%# Render target for workers. %>
+      <div id="nodes">
+        <%# Render target for nodes and workers. %>
       </div>
     </div>
     
     <div id="graphs">
       <div class="graph_container">
-        <div class="graph_title">Work Units in Queue:</div>
+        <div class="graph_title">Work Units in Queue</div>
         <div id="work_units_graph" class="graph"></div>
       </div>
       <div class="graph_container">
-        <div class="graph_title">Jobs in Queue:</div>
+        <div class="graph_title">Jobs in Queue</div>
         <div id="jobs_graph" class="graph"></div>
       </div>
       <div class="graph_container">
-        <div class="graph_title">Active Worker Daemons:</div>
+        <div class="graph_title">
+          Active Nodes
+          (<span id="nodes_legend" class="legend_box"></span>)
+          and Workers
+          (<span id="workers_legend" class="legend_box"></span>)
+        </div>
         <div id="workers_graph" class="graph"></div>
       </div>
     </div>
@@ -67,7 +72,7 @@
           <div id="worker_job_id">job #<span class="job_id"></span> / work unit #<span class="work_unit_id"></span></div>
         </div>
         <div id="worker_sleeping">
-          worker is sleeping&hellip;
+          worker exiting&hellip;
         </div>
       </div>
     </div>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: cloud-crowd
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Jeremy Ashkenas
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-14 00:00:00 -04:00
+date: 2009-09-17 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -63,14 +63,14 @@ dependencies:
         version: 1.10.0
     version: 
 - !ruby/object:Gem::Dependency 
-  name: daemons
+  name: thin
   type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.0.10
+        version: 1.2.4
     version: 
 - !ruby/object:Gem::Dependency 
   name: faker
@@ -144,23 +144,22 @@ files:
 - examples/word_count_example.rb
 - lib/cloud-crowd.rb
 - lib/cloud_crowd/action.rb
-- lib/cloud_crowd/app.rb
 - lib/cloud_crowd/asset_store/filesystem_store.rb
 - lib/cloud_crowd/asset_store/s3_store.rb
 - lib/cloud_crowd/asset_store.rb
 - lib/cloud_crowd/command_line.rb
-- lib/cloud_crowd/daemon.rb
 - lib/cloud_crowd/exceptions.rb
 - lib/cloud_crowd/helpers/authorization.rb
 - lib/cloud_crowd/helpers/resources.rb
 - lib/cloud_crowd/helpers.rb
 - lib/cloud_crowd/inflector.rb
 - lib/cloud_crowd/models/job.rb
+- lib/cloud_crowd/models/node_record.rb
 - lib/cloud_crowd/models/work_unit.rb
-- lib/cloud_crowd/models/worker_record.rb
 - lib/cloud_crowd/models.rb
-- lib/cloud_crowd/runner.rb
+- lib/cloud_crowd/node.rb
 - lib/cloud_crowd/schema.rb
+- lib/cloud_crowd/server.rb
 - lib/cloud_crowd/worker.rb
 - LICENSE
 - public/css/admin_console.css
@@ -171,6 +170,8 @@ files:
 - public/images/header_back.png
 - public/images/logo.png
 - public/images/queue_fill.png
+- public/images/server.png
+- public/images/server_busy.png
 - public/images/server_error.png
 - public/images/sidebar_bottom.png
 - public/images/sidebar_top.png
@@ -181,7 +182,7 @@ files:
 - public/js/flot.js
 - public/js/jquery.js
 - README
-- test/acceptance/test_app.rb
+- test/acceptance/test_server.rb
 - test/acceptance/test_failing_work_units.rb
 - test/acceptance/test_word_count.rb
 - test/blueprints.rb
@@ -194,7 +195,7 @@ files:
 - test/unit/test_configuration.rb
 - test/unit/test_job.rb
 - test/unit/test_work_unit.rb
-- views/index.erb
+- views/operations_center.erb
 has_rdoc: true
 homepage: http://wiki.github.com/documentcloud/cloud-crowd
 licenses: []

data/lib/cloud_crowd/daemon.rb

@@ -1,95 +0,0 @@
-CloudCrowd.configure(ENV['CLOUD_CROWD_CONFIG'])
-
-module CloudCrowd
-  
-  # A CloudCrowd::Daemon, started by the Daemons gem, runs a CloudCrowd::Worker in
-  # a loop, continually fetching and processing WorkUnits from the central
-  # server. 
-  # 
-  # The Daemon backs off and pings the central server less frequently 
-  # when there isn't any work to be done, and speeds back up when there is.
-  #
-  # The `crowd` command responds to all the usual methods that the Daemons gem
-  # supports.
-  class Daemon
-    
-    # The back-off factor used to slow down requests for new work units 
-    # when the queue is empty.
-    WAIT_MULTIPLIER   = 1.5
-    
-    MIN_WAIT = CloudCrowd.config[:min_worker_wait]
-    MAX_WAIT = CloudCrowd.config[:max_worker_wait]
-    
-    def initialize
-      @wait_time  = MIN_WAIT
-      @worker     = Worker.new
-      Signal.trap('INT')  { kill_worker_and_exit }
-      Signal.trap('KILL') { kill_worker_and_exit }
-      Signal.trap('TERM') { kill_worker_and_exit }
-    end
-    
-    # Spin up our worker and monitoring threads. The monitor's the boss, and 
-    # will feel no compunction in killing the worker thread if necessary.
-    # Check in before starting up. If check in fails, there's no sense in going.
-    def run
-      @worker.check_in('starting')
-      @work_thread = run_worker
-      @monitor_thread = run_monitor
-      @monitor_thread.join
-    end
-    
-    
-    private
-    
-    # Loop forever, fetching WorkUnits and processing them.
-    def run_worker
-      Thread.new do
-        loop do
-          @worker.fetch_work_unit
-          if @worker.has_work?
-            @wait_time = MIN_WAIT
-            while @worker.has_work?
-              @worker.run
-              sleep 0.01 # So as to listen for incoming signals.
-            end
-          else
-            @wait_time = [@wait_time * WAIT_MULTIPLIER, MAX_WAIT].min
-            sleep @wait_time
-          end
-        end
-      end
-    end
-    
-    # Checks in to let the central server know it's still alive every 
-    # CHECK_IN_INTERVAL seconds. Restarts the work_thread if it has died.
-    def run_monitor
-      Thread.new do
-        sleep Worker::CHECK_IN_INTERVAL
-        loop do
-          @work_thread = run_monitor unless @work_thread.alive? || @exit_started
-          @worker.check_in(@work_thread.status)
-          sleep Worker::CHECK_IN_INTERVAL
-        end
-      end
-    end
-    
-    def running?
-      @work_thread.alive? || @monitor_thread.alive?
-    end
-    
-    # At exit, kill the worker thread, gently at first, then forcefully.
-    def kill_worker_and_exit
-      @worker.check_out
-      @exit_started = Time.now
-      @work_thread.kill && @monitor_thread.kill
-      sleep 0.3 while running? && Time.now - @exit_started < WORKER_EXIT_WAIT
-      return Process.exit unless running?
-      @work_thread.kill! && @monitor_thread.kill!
-      Process.exit
-    end
-    
-  end
-  
-end
-
-CloudCrowd::Daemon.new.run

data/lib/cloud_crowd/models/worker_record.rb

@@ -1,61 +0,0 @@
-module CloudCrowd
-
-  # A WorkerRecord is a recording of an active worker daemon running remotely.
-  # Every time it checks in, we keep track of its status. The attributes shown
-  # here may lag their actual values by up to Worker::CHECK_IN_INTERVAL seconds.
-  class WorkerRecord < ActiveRecord::Base
-        
-    EXPIRES_AFTER = 2 * Worker::CHECK_IN_INTERVAL
-    
-    has_one :work_unit
-    
-    validates_presence_of :name, :thread_status
-    
-    before_destroy :clear_work_units
-    
-    named_scope :alive, lambda { {:conditions => ['updated_at > ?', Time.now - EXPIRES_AFTER]} }
-    named_scope :dead,  lambda { {:conditions => ['updated_at <= ?', Time.now - EXPIRES_AFTER]} }
-    
-    # Save a Worker's current status to the database.
-    def self.check_in(params)
-      attrs = {:thread_status => params[:thread_status], :updated_at => Time.now}
-      self.find_or_create_by_name(params[:name]).update_attributes!(attrs)
-    end
-    
-    # Remove a terminated Worker's record from the database.
-    def self.check_out(params)
-      self.find_by_name(params[:name]).destroy
-    end 
-    
-    # We consider the worker to be alive if it's checked in more recently
-    # than twice the expected interval ago.
-    def alive?
-      updated_at > Time.now - EXPIRES_AFTER
-    end
-    
-    # Derive the Worker's PID on the remote machine from the name.
-    def pid
-      @pid ||= self.name.split('@').first
-    end
-    
-    # Derive the hostname from the Worker's name.
-    def hostname
-      @hostname ||= self.name.split('@').last
-    end
-    
-    def to_json(opts={})
-      {
-        'name'    => name, 
-        'status'  => work_unit && work_unit.display_status,
-      }.to_json
-    end
-    
-    
-    private
-    
-    def clear_work_units
-      WorkUnit.update_all('worker_record_id = null', "worker_record_id = #{id}")
-    end
-    
-  end
-end