cloud-crowd 0.7.3 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cca46d87f48a34b124b8b581e0f65f3d68b5595b
-  data.tar.gz: d0b1df360b9a5cd87bb3183b3c49480997890e71
+  metadata.gz: 82aa3ec5f4de9712060de2ba1f62207cc4c9d57c
+  data.tar.gz: 8129d294659a958f2ae1031dba8f4b507d064d03
 SHA512:
-  metadata.gz: 1cf7261ba8b1333302a42793a0d2c18f4c00432a5e2b58c6ad3f01526895e985b8c82da1779d96bb400c1c182529f8e3150a82221a2543141d296c9e1e977b82
-  data.tar.gz: bebaf475fcc3ccce3d96db36658dc7d16e6fb3b9378788ae2c3623953d646792484cef5c3a562e3fdad3b97882244830419d0da93e5d09f87212a9a4738f1aac
+  metadata.gz: 45c9bf5f14369c0b9d30d8eb2e6cc3dcbf6bcf046ccff6e6262682ba94b5bb26b3d23fd4fe4c8c1e7f7a980146abad9f12d5fa3baeeace408e6f2e6ca9db0c38
+  data.tar.gz: a4c1687bf3c7ada3354c30a19eaa48d8709f1c2a9bd3b874e4d823519ee3201600d9f20607d003dc7430c188395474d0ffe52d0e36e900e8f2c1c530b96b1111

data/cloud-crowd.gemspec

@@ -1,7 +1,11 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cloud_crowd/version'
+
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.7.3'         # Keep version in sync with cloud-cloud.rb
-  s.date      = '2014-05-06'
+  s.version   = CloudCrowd::VERSION
+  s.date      = CloudCrowd::VERSION_RELEASED
 
   s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
   s.summary     = "Parallel Processing for the Rest of Us"

data/lib/cloud-crowd.rb

@@ -25,6 +25,7 @@ autoload :YAML,         'yaml'
 # Common code which should really be required in every circumstance.
 require 'socket'
 require 'net/http'
+require 'cloud_crowd/version'
 require 'cloud_crowd/exceptions'
 require 'rest_client'
 require 'pathname'
@@ -45,9 +46,7 @@ module CloudCrowd
   autoload :Server,       'cloud_crowd/server'
   autoload :Worker,       'cloud_crowd/worker'
   autoload :WorkUnit,     'cloud_crowd/models'
-
-  # Keep this version in sync with the gemspec.
-  VERSION        = '0.7.2'
+  autoload :Dispatcher,   'cloud_crowd/dispatcher'
 
   # Increment the schema version when there's a backwards incompatible change.
   SCHEMA_VERSION = 4
@@ -113,7 +112,11 @@ module CloudCrowd
       configuration = YAML.load(ERB.new(File.read(config_path)).result)
       ActiveRecord::Base.establish_connection(configuration)
       if validate_schema
-        version = ActiveRecord::Base.connection.select_values('select max(version) from schema_migrations').first.to_i
+        begin
+          version = ActiveRecord::Base.connection.select_values('select max(version) from schema_migrations').first.to_i
+        rescue
+          version = 0
+        end
         return true if version == SCHEMA_VERSION
         puts "Your database schema is out of date. Please use `crowd load_schema` to update it. This will wipe all the tables, so make sure that your jobs have a chance to finish first.\nexiting..."
         exit
@@ -208,6 +211,12 @@ module CloudCrowd
       @identity == :node
     end
 
+    # Output a message with the current Timestamp prepended.
+    # Sinatra will re-direct stdout to a log file located at "log_path"
+    def log(message)
+      printf("%-20s %s\n", Time.now.strftime("%F-%T:"), message)
+    end
+
   end
 
 end

data/lib/cloud_crowd/command_line.rb

@@ -83,7 +83,7 @@ Options:
       pid_path    = CloudCrowd.pid_path('server.pid')
       rackup_path = File.expand_path("#{@options[:config_path]}/config.ru")
       FileUtils.mkdir_p(CloudCrowd.log_path) if @options[:daemonize] && !File.exists?(CloudCrowd.log_path)
-      puts "Starting CloudCrowd Central Server on port #{port}..."
+      puts "Starting CloudCrowd Central Server (#{VERSION}) on port #{port}..."
       exec "thin -e #{@options[:environment]} -p #{port} #{daemonize} --tag cloud-crowd-server --log #{log_path} --pid #{pid_path} -R #{rackup_path} start"
     end
 
@@ -114,7 +114,7 @@ Options:
     # will be long-lived, although its workers will come and go.
     def start_node
       @options[:port] ||= Node::DEFAULT_PORT
-      puts "Starting CloudCrowd Node on port #{@options[:port]}..."
+      puts "Starting CloudCrowd Node (#{VERSION}) on port #{@options[:port]}..."
       Node.new(@options)
     end
 
@@ -242,4 +242,4 @@ Options:
     end
 
   end
-end
+end

data/lib/cloud_crowd/dispatcher.rb

@@ -0,0 +1,52 @@
+module CloudCrowd
+
+  # The dispatcher is responsible for distributing work_units
+  # to the worker nodes.
+  #
+  # It automatically performs the distribution on a set schedule,
+  # but can also be signaled to perform distribution immediately
+  class Dispatcher
+
+    # Starts distributing jobs every "distribution_interval" seconds
+    def initialize(distribution_interval)
+      @mutex  = Mutex.new
+      @awaken = ConditionVariable.new
+      distribute_periodically(distribution_interval)
+    end
+
+    # Sends a signal to the distribution thread.
+    # If it's asleep, it will wake up and perform a distribution.
+    def distribute!
+      @mutex.synchronize do
+        @awaken.signal
+      end
+    end
+
+    private
+
+    # Perform distribution of work units in a background thread
+    def distribute_periodically(interval)
+      Thread.new{
+        loop do
+          perform_distribution
+          # Sleep for "interval" seconds.
+          # If awaken isn't signaled, timeout and attempt distribution
+          @mutex.synchronize do
+            @awaken.wait(@mutex, interval)
+          end
+        end
+      }
+    end
+
+    def perform_distribution
+      #CloudCrowd.log "Distributing jobs to nodes"
+      begin
+        WorkUnit.distribute_to_nodes
+      rescue StandardError => e
+        CloudCrowd.log "Exception: #{e}"
+        CloudCrowd.log e.backtrace
+      end
+    end
+
+  end
+end

data/lib/cloud_crowd/models/job.rb

@@ -53,7 +53,7 @@ module CloudCrowd
       return queue_for_workers([outs]) if merging?
       if complete?
         update_attributes(:outputs => outs, :time => time_taken)
-        puts "Job ##{id} (#{action}) #{display_status}." unless ENV['RACK_ENV'] == 'test'
+        CloudCrowd.log "Job ##{id} (#{action}) #{display_status}." unless ENV['RACK_ENV'] == 'test'
         CloudCrowd.defer { fire_callback } if callback_url
       end
       self
@@ -82,7 +82,7 @@ module CloudCrowd
         response = RestClient.post(callback_url, {:job => self.to_json})
         CloudCrowd.defer { self.destroy } if response && response.code == 201
       rescue RestClient::Exception => e
-        puts "Job ##{id} (#{action}) failed to fire callback: #{callback_url}"
+        CloudCrowd.log "Job ##{id} (#{action}) failed to fire callback: #{callback_url}"
       end
     end
 

data/lib/cloud_crowd/models/node_record.rb

@@ -32,7 +32,13 @@ module CloudCrowd
         :max_workers      => params[:max_workers],
         :enabled_actions  => params[:enabled_actions]
       }
-      self.find_or_create_by(:host => params[:host]).update_attributes!(attrs)
+      host_attr = {:host => params[:host]}
+      if (record = where(host_attr).first)
+        record.update_attributes!(attrs)
+        record
+      else
+        create!(attrs.merge(host_attr))
+      end
     end
 
     # Dispatch a WorkUnit to this node. Places the node at back at the end of
@@ -46,14 +52,14 @@ module CloudCrowd
       touch && true
     rescue RestClient::RequestTimeout
       # The node's gone away.  Destroy it and it will check in when it comes back
-      puts "Node #{host} received RequestTimeout, removing it"
+      CloudCrowd.log "Node #{host} received RequestTimeout, removing it"
       destroy && false
     rescue RestClient::RequestFailed => e
       raise e unless e.http_code == 503 && e.http_body == Node::OVERLOADED_MESSAGE
       update_attribute(:busy, true) && false
     rescue RestClient::Exception, Errno::ECONNREFUSED, Timeout::Error, Errno::ECONNRESET=>e
       # Couldn't post to node, assume it's gone away.
-      puts "Node #{host} received #{e.class} #{e}, removing it"
+      CloudCrowd.log "Node #{host} received #{e.class} #{e}, removing it"
       destroy && false
     end
 
@@ -121,7 +127,6 @@ module CloudCrowd
     # Redistribute in a separate thread to avoid delaying shutdown.
     def redistribute_work_units
       release_work_units
-      CloudCrowd.defer { WorkUnit.distribute_to_nodes }
     end
 
   end

data/lib/cloud_crowd/models/work_unit.rb

@@ -79,7 +79,7 @@ module CloudCrowd
       reservation = SecureRandom.random_number(MAX_RESERVATION)
       conditions = "reservation is null and node_record_id is null and status in (#{INCOMPLETE.join(',')}) and #{options[:conditions]}"
       query = WorkUnit.where(conditions)
-      query.limit(options[:limit]) if options[:limit]
+      query = query.limit(options[:limit]) if options[:limit]
       any = query.update_all("reservation = #{reservation}") > 0
       any && reservation
     end

data/lib/cloud_crowd/node.rb

@@ -59,7 +59,8 @@ module CloudCrowd
       throw :halt, [503, OVERLOADED_MESSAGE] if @overloaded
       unit = JSON.parse(params[:work_unit])
       pid = fork { Worker.new(self, unit).run }
-      Process.detach(pid)
+      thread = Process.detach(pid)
+      track_work(unit["id"], thread)
       json :pid => pid
     end
 
@@ -78,6 +79,7 @@ module CloudCrowd
       @overloaded       = false
       @max_load         = CloudCrowd.config[:max_load]
       @min_memory       = CloudCrowd.config[:min_free_memory]
+      @work             = {}
       start unless ENV['RACK_ENV'] == 'test'
     end
 
@@ -92,7 +94,7 @@ module CloudCrowd
       @server.daemonize if @daemon
       trap_signals
       asset_store
-      @server_thread   = CloudCrowd.defer { @server.start }
+      @server_thread   = Thread.new { @server.start }
       check_in(true)
       check_in_periodically
       monitor_system if @max_load || @min_memory
@@ -110,7 +112,7 @@ module CloudCrowd
         :enabled_actions  => @enabled_actions.join(',')
       )
     rescue RestClient::Exception, Errno::ECONNREFUSED
-      puts "Failed to connect to the central server (#{@central.to_s})."
+      CloudCrowd.log "Failed to connect to the central server (#{@central.to_s})."
       raise SystemExit if critical
     end
 
@@ -151,15 +153,44 @@ module CloudCrowd
         raise NotImplementedError, "'min_free_memory' is not yet implemented on your platform"
       end
     end
+    
+    def track_work(id, thread)
+      @work[id] = { thread: thread, start: Time.now }
+    end
 
-
+    def check_on_workers
+      # ToDo, this isn't really thread safe.
+      # there are events in which a job completes and exits successfully
+      # while iteration here is taking place.  However the interleaving
+      # is such that the work unit should be complete / cleaned up already
+      # even in the event that a thread is flagged as dead here.
+      @work.each do |unit_id, work|
+        unless work[:thread].alive?
+          CloudCrowd.log "Notifying central server that worker #{work[:thread].pid} for unit #{unit_id} mysteriously died."
+          data = {
+            id:     unit_id,
+            pid:    work[:thread].pid,
+            status: 'failed',
+            output: { output: "Worker thread #{work[:thread].pid} died on #{host} prior to #{Time.now}" }.to_json,
+            time:   Time.now - work[:start] # this is time until failure was noticed
+          }
+          @central["/work/#{unit_id}"].put(data)
+          resolve_work(unit_id)
+        end
+      end
+    end
+    
+    def resolve_work(unit_id)
+      @work.delete(unit_id)
+    end
+    
     private
 
     # Launch a monitoring thread that periodically checks the node's load
     # average and the amount of free memory remaining. If we transition out of
     # the overloaded state, let central know.
     def monitor_system
-      @monitor_thread = CloudCrowd.defer do
+      @monitor_thread = Thread.new do
         loop do
           was_overloaded = @overloaded
           @overloaded = overloaded?
@@ -173,8 +204,9 @@ module CloudCrowd
     # will assume that the node has gone down. Checking in will let central know
     # it's still online.
     def check_in_periodically
-      @check_in_thread = CloudCrowd.defer do
+      @check_in_thread = Thread.new do
         loop do
+          check_on_workers
           reply = ""
           1.upto(5).each do | attempt_number |
             # sleep for an ever increasing amount of time to prevent overloading the server
@@ -183,13 +215,13 @@ module CloudCrowd
             # if we did not receive a reply, the server has went away; it
             # will reply with an empty string if the check-in succeeds
             if reply.nil?
-              puts "Failed on attempt # #{attempt_number} to check in with server"
+              CloudCrowd.log "Failed on attempt ##{attempt_number} to check in with server"
             else
               break
             end
           end
           if reply.nil?
-            puts "Giving up after repeated attempts to contact server"
+            CloudCrowd.log "Giving up after repeated attempts to contact server"
             raise SystemExit
           end
         end
@@ -200,7 +232,6 @@ module CloudCrowd
     def trap_signals
       Signal.trap('QUIT') { shut_down }
       Signal.trap('INT')  { shut_down }
-      Signal.trap('KILL') { shut_down }
       Signal.trap('TERM') { shut_down }
     end
 

data/lib/cloud_crowd/server.rb

@@ -20,6 +20,10 @@ module CloudCrowd
   class Server < Sinatra::Base
     use ActiveRecord::ConnectionAdapters::ConnectionManagement
 
+    # The interval (in seconds) at which the server will distribute
+    # new work units to the nodes
+    DISTRIBUTE_INTERVAL=20
+
     set :root, ROOT
     set :authorization_realm, "CloudCrowd"
 
@@ -54,11 +58,6 @@ module CloudCrowd
       `tail -n 100 #{CloudCrowd.log_path('server.log')}`
     end
 
-    # Get the JSON for what a worker is up to.
-    get '/worker/:name' do
-      json WorkUnit.find_by_worker_name(params[:name]) || {}
-    end
-
     # To monitor the central server with Monit, God, Nagios, or another
     # monitoring tool, you can hit /heartbeat to make sure.
     get '/heartbeat' do
@@ -71,8 +70,8 @@ module CloudCrowd
     # Distributes all work units to available nodes.
     post '/jobs' do
       job = Job.create_from_request(JSON.parse(params[:job]))
-      CloudCrowd.defer { WorkUnit.distribute_to_nodes }
-      puts "Job ##{job.id} (#{job.action}) started." unless ENV['RACK_ENV'] == 'test'
+      CloudCrowd.log("Job ##{job.id} (#{job.action}) started.") unless ENV['RACK_ENV'] == 'test'
+      @dispatcher.distribute!
       json job
     end
 
@@ -95,11 +94,8 @@ module CloudCrowd
     # configuration with the central server. Triggers distribution of WorkUnits.
     put '/node/:host' do
       NodeRecord.check_in(params, request)
-      puts "Node #{params[:host]} checked in."
-      CloudCrowd.defer do
-        sleep 15 # Give the new node awhile to start listening
-        WorkUnit.distribute_to_nodes
-      end
+      CloudCrowd.log "Node #{params[:host]} checked in."
+      @dispatcher.distribute!
       json nil
     end
 
@@ -107,7 +103,8 @@ module CloudCrowd
     # WorkUnits it may have had checked out.
     delete '/node/:host' do
       NodeRecord.destroy_all(:host => params[:host])
-      puts "Node #{params[:host]} checked out."
+      CloudCrowd.log "Node #{params[:host]} checked out."
+      @dispatcher.distribute!
       json nil
     end
 
@@ -115,20 +112,22 @@ module CloudCrowd
     # they mark it back on the central server and exit. Triggers distribution
     # of pending work units.
     put '/work/:work_unit_id' do
+      CloudCrowd.log "Job #{current_work_unit.job_id} WorkUnit #{current_work_unit.id} #{current_work_unit.action} #{params[:status]} in #{params[:time]}"
       case params[:status]
       when 'succeeded' then current_work_unit.finish(params[:output], params[:time])
       when 'failed'    then current_work_unit.fail(params[:output], params[:time])
       else             error(500, "Completing a work unit must specify status.")
       end
-      CloudCrowd.defer { WorkUnit.distribute_to_nodes }
-
+      @dispatcher.distribute!
       json nil
     end
 
     # At initialization record the identity of this Ruby instance as a server.
     def initialize(*args)
+      CloudCrowd.log "Starting server"
       super(*args)
       CloudCrowd.identity = :server
+      @dispatcher = Dispatcher.new(DISTRIBUTE_INTERVAL)
     end
 
   end

data/lib/cloud_crowd/version.rb

@@ -0,0 +1,4 @@
+module CloudCrowd
+  VERSION          = '0.7.5'
+  VERSION_RELEASED = '2015-06-17'
+end

data/lib/cloud_crowd/worker.rb

@@ -89,6 +89,7 @@ module CloudCrowd
         action.cleanup_work_directory if action
         fail_work_unit(e)
       end
+      @node.resolve_work(@unit['id'])
     end
 
     # Run this worker inside of a fork. Attempts to exit cleanly.
@@ -141,7 +142,6 @@ module CloudCrowd
     def trap_signals
       Signal.trap('QUIT') { Process.exit! }
       Signal.trap('INT')  { Process.exit! }
-      Signal.trap('KILL') { Process.exit! }
       Signal.trap('TERM') { Process.exit! }
     end
 

data/test/acceptance/test_failing_work_units.rb

@@ -4,9 +4,10 @@ require 'test_helper'
 class FailingWorkUnitsTest < Minitest::Test
 
   should "retry work units when they fail" do
-    WorkUnit.expects(:distribute_to_nodes).returns(true)
+    WorkUnit.stubs(:distribute_to_nodes).returns([])
+    Dispatcher.any_instance.stubs(:distribute_periodically)
+    Dispatcher.any_instance.expects(:distribute!)
     browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
-    
     browser.post '/jobs', :job => {
       'action'  => 'failure_testing',
       'inputs'  => ['one', 'two', 'three'],

data/test/acceptance/test_server.rb

@@ -11,10 +11,13 @@ class ServerTest < Minitest::Test
   context "The CloudCrowd::Server (Sinatra)" do
 
     setup do
+      WorkUnit.stubs(:distribute_to_nodes).returns([])
+      Dispatcher.any_instance.stubs(:distribute_periodically)
       Job.destroy_all
       2.times { Job.make! }
     end
 
+
     should "set the identity of the Ruby instance" do
       app.new
       assert CloudCrowd.server?
@@ -37,7 +40,7 @@ class ServerTest < Minitest::Test
     end
 
     should "be able to create a job" do
-      WorkUnit.expects(:distribute_to_nodes).returns(true)
+      Dispatcher.any_instance.expects(:distribute!)
       post('/jobs', :job => '{"action":"graphics_magick","inputs":["http://www.google.com/"]}')
       assert last_response.ok?
       job_info = JSON.parse(last_response.body)

data/test/acceptance/test_word_count.rb

@@ -5,7 +5,8 @@ class WordCountTest < Minitest::Test
   context "the word_count action" do
     
     setup do
-      WorkUnit.expects(:distribute_to_nodes).returns(true)
+      Dispatcher.any_instance.expects(:distribute!)
+      Dispatcher.any_instance.stubs(:distribute_periodically)
       @asset_store = AssetStore.new
       @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
       @browser.put('/worker', :name => 'test_worker', :thread_status => 'sleeping')
@@ -16,14 +17,13 @@ class WordCountTest < Minitest::Test
     should "be able to create a word_count job" do
       assert @browser.last_response.ok? 
       info = JSON.parse(@browser.last_response.body)
-      assert info['status'] == 'processing'
-      assert info['work_units'] == 1
+      assert_equal 'processing', info['status']
+      assert_equal 1, info['work_units']
     end
     
     should "be able to perform the processing stage of a word_count" do
       action = CloudCrowd.actions['word_count'].new(1, "file://#{File.expand_path(__FILE__)}", {}, @asset_store)
-      count = action.process
-      assert count == 101
+      assert_equal 96, action.process
     end
     
   end

data/test/test_helper.rb

@@ -19,10 +19,25 @@ CloudCrowd.configure_database(here + '/config/database.yml')
 
 require "#{CloudCrowd::ROOT}/test/blueprints.rb"
 
+
+module TestHelpers
+  def setup
+    CloudCrowd::WorkUnit.stubs(:distribute_to_nodes).returns([])
+    CloudCrowd.stubs(:log)
+    super
+  end
+  def teardown
+      Mocha::Mockery.instance.teardown
+      Mocha::Mockery.reset_instance
+      super
+  end
+end
+
 class Minitest::Test
-  include CloudCrowd
+  include TestHelpers
   include Shoulda::Matchers::ActiveRecord
   extend Shoulda::Matchers::ActiveRecord
   include Shoulda::Matchers::ActiveModel
   extend Shoulda::Matchers::ActiveModel
+  include CloudCrowd
 end

data/test/unit/test_job.rb

@@ -77,6 +77,7 @@ class JobTest < Minitest::Test
       Job.any_instance.stubs(:fire_callback).returns(true)
       Job.any_instance.expects(:fire_callback)
       @job.work_units.first.finish('{"output":"output"}', 10)
+      sleep 0.5 # block to allow Crowd.defer thread to execute
       assert @job.all_work_units_complete?
     end
     

data/test/unit/test_node_record.rb

@@ -39,6 +39,24 @@ class NodeRecordTest < Minitest::Test
       assert !!URI.parse(@node.url)
     end
     
+    should "be able to check-in and be updated" do
+      request = Rack::Request.new({'REMOTE_ADDR'=>'127.0.0.1'})
+      node_data = {
+          :ip_address      => '127.0.0.1',
+          :host            => "hostname-42:6032",
+          :busy            => false,
+          :max_workers     => 3,
+          :enabled_actions => 'graphics_magick,word_count'
+      }
+      node_data[:host] << ':6093'
+      record = NodeRecord.check_in( node_data, request )
+      assert_equal '127.0.0.1', record.ip_address
+      assert_equal 3, record.max_workers
+      node_data[:max_workers] = 2
+      updated_record = NodeRecord.check_in( node_data, request )
+      assert_equal updated_record, record
+      assert_equal 2, updated_record.max_workers
+    end
   end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-crowd
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.5
 platform: ruby
 authors:
 - Jeremy Ashkenas
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-06 00:00:00.000000000 Z
+date: 2015-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -214,6 +214,7 @@ files:
 - lib/cloud_crowd/asset_store/filesystem_store.rb
 - lib/cloud_crowd/asset_store/s3_store.rb
 - lib/cloud_crowd/command_line.rb
+- lib/cloud_crowd/dispatcher.rb
 - lib/cloud_crowd/exceptions.rb
 - lib/cloud_crowd/helpers.rb
 - lib/cloud_crowd/helpers/authorization.rb
@@ -226,6 +227,7 @@ files:
 - lib/cloud_crowd/node.rb
 - lib/cloud_crowd/schema.rb
 - lib/cloud_crowd/server.rb
+- lib/cloud_crowd/version.rb
 - lib/cloud_crowd/worker.rb
 - public/css/admin_console.css
 - public/css/reset.css