documentcloud-cloud-crowd 0.0.5 → 0.0.6

data/lib/cloud_crowd/app.rb

@@ -4,7 +4,7 @@ module CloudCrowd
   #
   # == Admin
   # [get /] Render the admin console, with a progress meter for running jobs.
-  # [get /jobs] Get the combined JSON of every active job in the queue.
+  # [get /status] Get the combined JSON of every active job and worker.
   # [get /heartbeat] Returns 200 OK to let monitoring tools know the server's up.
   # 
   # == Public API
@@ -15,6 +15,7 @@ module CloudCrowd
   # == Internal Workers API
   # [post /work] Dequeue the next WorkUnit, and hand it off to the worker.
   # [put /work/:unit_id] Mark a finished WorkUnit as completed or failed, with results.
+  # [put /worker] Keep a record of an actively running worker.
   class App < Sinatra::Default
     
     set :root, ROOT
@@ -35,9 +36,21 @@ module CloudCrowd
       erb :index
     end
     
-    # Get the JSON for every active job in the queue.
-    get '/jobs' do
-      json Job.incomplete
+    # Get the JSON for every active job in the queue and every active worker
+    # in the system. This action may get a little worrisome as the system grows
+    # larger -- keep it in mind.
+    get '/status' do
+      json(
+        'jobs'            => Job.incomplete, 
+        'workers'         => WorkerRecord.alive(:order => 'name desc'),
+        'work_unit_count' => WorkUnit.incomplete.count
+      )
+    end
+    
+    # Get the JSON for a worker record's work unit, if one exists.
+    get '/worker/:name' do
+      record = WorkerRecord.find_by_name params[:name]
+      json((record && record.work_unit) || {})
     end
     
     # To monitor the central server with Monit, God, Nagios, or another 
@@ -66,6 +79,8 @@ module CloudCrowd
       json nil
     end
     
+    # INTERNAL WORKER DAEMON API:
+    
     # Internal method for worker daemons to fetch the work unit at the front
     # of the queue. Work unit is marked as taken and handed off to the worker.
     post '/work' do
@@ -90,6 +105,13 @@ module CloudCrowd
       end
     end
     
+    # Every so often workers check in to let the central server know that
+    # they're still alive. Keep up-to-date records
+    put '/worker' do
+      params[:terminated] ? WorkerRecord.check_out(params) : WorkerRecord.check_in(params)
+      json nil
+    end
+    
   end
   
 end

data/lib/cloud_crowd/asset_store.rb

@@ -9,14 +9,20 @@ module CloudCrowd
   # You shouldn't need to use the AssetStore directly -- Action's +download+
   # and +save+ methods use it behind the scenes.
   class AssetStore
-    include FileUtils
     
-    # Creating an AssetStore will determine wether to save private or public
-    # files on S3, depending on the value of <tt>use_s3_authentication</tt> in 
-    # <tt>config.yml</tt>.
+    LOCAL_STORAGE_PATH = '/tmp/cloud_crowd_storage'
+    
+    # Creating an AssetStore mixes in the specific storage implementation 
+    # specified by 'storage' in <tt>config.yml</tt>.
     def initialize
       @use_auth = CloudCrowd.config[:use_s3_authentication]
-      mkdir_p temp_storage_path unless File.exists? temp_storage_path
+      @storage  = CloudCrowd.config[:storage]
+      FileUtils.mkdir_p temp_storage_path unless File.exists? temp_storage_path
+      case @storage
+      when 's3'         then extend S3Store
+      when 'filesystem' then extend FilesystemStore
+      else raise StorageNotFound, "#{@storage} is not a valid storage back end"
+      end
     end
     
     # Get the path to CloudCrowd's temporary local storage. All actions run
@@ -25,45 +31,68 @@ module CloudCrowd
       "#{Dir.tmpdir}/cloud_crowd_tmp"
     end
     
-    # Copy a finished file from our local storage to S3. Save it publicly unless
-    # we're configured to use S3 authentication.
-    def save(local_path, save_path)
-      ensure_s3_connection
-      permission = @use_auth ? 'private' : 'public-read'
-      @bucket.put(save_path, File.open(local_path), {}, permission)
-    end
     
-    # Cleanup all S3 files for a job that's been completed and retrieved.
-    def cleanup_job(job)
-      ensure_s3_connection
-      @bucket.delete_folder("#{job.action}/job_#{job.id}")
-    end
-    
-    # Return the S3 public URL for a finshed file. Authenticated links expire
-    # after one day by default.
-    def url(save_path)
-      @use_auth ? @s3.interface.get_link(@bucket, save_path) :
-                  @bucket.key(save_path).public_link
+    # The S3Store is an implementation of an AssetStore that uses a bucket
+    # on S3 for all resulting files.
+    module S3Store
+      
+      # Save a finished file from local storage to S3. Save it publicly unless 
+      # we're configured to use S3 authentication.
+      def save(local_path, save_path)
+        ensure_s3_connection
+        permission = @use_auth ? 'private' : 'public-read'
+        @bucket.put(save_path, File.open(local_path), {}, permission)
+      end
+      
+      # Return the S3 public URL for a finshed file. Authenticated links expire
+      # after one day by default.
+      def url(save_path)
+        @use_auth ? @s3.interface.get_link(@bucket, save_path) :
+                    @bucket.key(save_path).public_link
+      end
+      
+      # Remove all of a Job's resulting files from S3, both intermediate and finished.
+      def cleanup_job(job)
+        ensure_s3_connection
+        @bucket.delete_folder("#{job.action}/job_#{job.id}")
+      end
+      
+      # Workers, through the course of many WorkUnits, keep around an AssetStore.
+      # Ensure we have a persistent S3 connection after first use.
+      def ensure_s3_connection
+        unless @s3 && @bucket
+          params = {:port => 80, :protocol => 'http'}
+          @s3 = RightAws::S3.new(CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key], params)
+          @bucket = @s3.bucket(CloudCrowd.config[:s3_bucket], true)
+        end
+      end
     end
     
-    private
-    
-    # Unused for the moment. Think about using the filesystem instead of S3
-    # in development.
-    def save_to_filesystem(local_path, save_path)
-      save_path = File.join("/tmp/cloud_crowd_storage", save_path)
-      save_dir = File.dirname(save_path)
-      mkdir_p save_dir unless File.exists? save_dir
-      cp(local_path, save_path)
-    end
     
-    # Workers, through the course of many WorkUnits, keep around an AssetStore.
-    # Ensure we have a persistent S3 connection after first use.
-    def ensure_s3_connection
-      unless @s3 && @bucket
-        params = {:port => 80, :protocol => 'http'}
-        @s3 = RightAws::S3.new(CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key], params)
-        @bucket = @s3.bucket(CloudCrowd.config[:s3_bucket], true)
+    # The FilesystemStore is an implementation of the AssetStore, good only for
+    # use in development, testing, or if you're only running a single-machine
+    # installation.
+    module FilesystemStore
+      
+      # Save a file to somewhere semi-persistent on the filesystem. Can be used
+      # in development, when offline, or if you happen to have a single-machine
+      # CloudCrowd installation. To use, configure :local_storage.
+      def save(local_path, save_path)
+        save_path = File.join(LOCAL_STORAGE_PATH, save_path)
+        save_dir = File.dirname(save_path)
+        FileUtils.mkdir_p save_dir unless File.exists? save_dir
+        FileUtils.cp(local_path, save_path)
+      end
+      
+      # Return the URL for a file saved to the local filesystem.
+      def url(save_path)
+        "file://#{File.expand_path(File.join(LOCAL_STORAGE_PATH, save_path))}"
+      end
+      
+      # Remove all of a Job's result files from the filesystem.
+      def cleanup_job(job)
+        path = "#{LOCAL_STORAGE_PATH}/#{job.action}/job_#{job.id}"
+        FileUtils.rm_r(path) if File.exists?(path)
       end
     end
   

data/lib/cloud_crowd/command_line.rb

@@ -14,18 +14,20 @@ module CloudCrowd
     
     # Command-line banner for the usage message.
     BANNER = <<-EOS
-CloudCrowd is a Ruby & AWS batch processing system, MapReduce style.    
+CloudCrowd is a MapReduce-inspired Parallel Processing System for Ruby.
+
+Wiki: http://wiki.github.com/documentcloud/cloud-crowd
 
 Usage: crowd COMMAND OPTIONS
 
-COMMANDS:
+Commands:
   install       Install the CloudCrowd configuration files to the specified directory
   server        Start up the central server (requires a database)
   workers       Control worker daemons, use: (start | stop | restart | status | run)
   console       Launch a CloudCrowd console, connected to the central database
   load_schema   Load the schema into the database specified by database.yml
 
-OPTIONS:
+Options:
     EOS
     
     # Creating a CloudCrowd::CommandLine runs from the contents of ARGV.
@@ -161,7 +163,7 @@ OPTIONS:
         opts.on('-p', '--port PORT', 'central server port number') do |port_num|
           @options[:port] = port_num
         end
-        opts.on('-e', '--environment ENV', 'Sinatra environment (code reloading)') do |env|
+        opts.on('-e', '--environment ENV', 'server environment (sinatra)') do |env|
           @options[:environment] = env
         end
         opts.on_tail('-v', '--version', 'show version') do

data/lib/cloud_crowd/daemon.rb

@@ -1,7 +1,5 @@
 CloudCrowd.configure(ENV['CLOUD_CROWD_CONFIG'])
 
-require 'cloud_crowd/worker'
-
 module CloudCrowd
   
   # A CloudCrowd::Daemon, started by the Daemons gem, runs a CloudCrowd::Worker in
@@ -15,39 +13,81 @@ module CloudCrowd
   # supports.
   class Daemon
     
-    MIN_WAIT        = CloudCrowd.config[:min_worker_wait]
-    MAX_WAIT        = CloudCrowd.config[:max_worker_wait]
-    WAIT_MULTIPLIER = CloudCrowd.config[:worker_wait_multiplier]
+    # The back-off factor used to slow down requests for new work units 
+    # when the queue is empty.
+    WAIT_MULTIPLIER   = 1.5
+    
+    MIN_WAIT = CloudCrowd.config[:min_worker_wait]
+    MAX_WAIT = CloudCrowd.config[:max_worker_wait]
     
     def initialize
-      @wait_time = MIN_WAIT
-      @worker = Worker.new
-      Signal.trap('INT',  'EXIT')
-      Signal.trap('KILL', 'EXIT')
-      Signal.trap('TERM', 'EXIT')
+      @wait_time  = MIN_WAIT
+      @worker     = Worker.new
+      Signal.trap('INT')  { kill_worker_and_exit }
+      Signal.trap('KILL') { kill_worker_and_exit }
+      Signal.trap('TERM') { kill_worker_and_exit }
     end
     
-    # Loop forever, fetching WorkUnits.
-    # TODO: Workers busy with their work units won't die until the unit has 
-    # been finished. This should probably be wrapped in an appropriately lengthy
-    # timeout, or should be killable from the outside by terminating the thread.
-    # In either case, nasty un-cleaned-up bits might be left behind.
+    # Spin up our worker and monitoring threads. The monitor's the boss, and 
+    # will feel no compunction in killing the worker thread if necessary.
+    # Check in before starting up. If check in fails, there's no sense in going.
     def run
-      loop do
-        @worker.fetch_work_unit
-        if @worker.has_work?
-          @wait_time = MIN_WAIT
-          while @worker.has_work?
-            @worker.run
-            sleep 0.01 # So as to listen for incoming signals.
+      @worker.check_in('starting')
+      @work_thread = run_worker
+      @monitor_thread = run_monitor
+      @monitor_thread.join
+    end
+    
+    
+    private
+    
+    # Loop forever, fetching WorkUnits and processing them.
+    def run_worker
+      Thread.new do
+        loop do
+          @worker.fetch_work_unit
+          if @worker.has_work?
+            @wait_time = MIN_WAIT
+            while @worker.has_work?
+              @worker.run
+              sleep 0.01 # So as to listen for incoming signals.
+            end
+          else
+            @wait_time = [@wait_time * WAIT_MULTIPLIER, MAX_WAIT].min
+            sleep @wait_time
           end
-        else
-          @wait_time = [@wait_time * WAIT_MULTIPLIER, MAX_WAIT].min
-          sleep @wait_time
         end
       end
     end
     
+    # Checks in to let the central server know it's still alive every 
+    # CHECK_IN_INTERVAL seconds. Restarts the work_thread if it has died.
+    def run_monitor
+      Thread.new do
+        sleep Worker::CHECK_IN_INTERVAL
+        loop do
+          @work_thread = run_monitor unless @work_thread.alive? || @exit_started
+          @worker.check_in(@work_thread.status)
+          sleep Worker::CHECK_IN_INTERVAL
+        end
+      end
+    end
+    
+    def running?
+      @work_thread.alive? || @monitor_thread.alive?
+    end
+    
+    # At exit, kill the worker thread, gently at first, then forcefully.
+    def kill_worker_and_exit
+      @worker.check_out
+      @exit_started = Time.now
+      @work_thread.kill && @monitor_thread.kill
+      sleep 0.3 while running? && Time.now - @exit_started < WORKER_EXIT_WAIT
+      return Process.exit unless running?
+      @work_thread.kill! && @monitor_thread.kill!
+      Process.exit
+    end
+    
   end
   
 end

data/lib/cloud_crowd/exceptions.rb

@@ -9,6 +9,11 @@ module CloudCrowd
   class ActionNotFound < Error #:nodoc:
   end
   
+  # StorageNotFound is raised when config.yml specifies a storage back end that
+  # doesn't exist.
+  class StorageNotFound < Error #:nodoc:
+  end
+  
   # StatusUnspecified is raised when a WorkUnit returns without a valid
   # status code.
   class StatusUnspecified < Error #:nodoc:

data/lib/cloud_crowd/helpers/resources.rb

@@ -24,8 +24,8 @@ module CloudCrowd
       # with no content.
       def dequeue_work_unit(offset=0)
         handle_conflicts do
-          actions = params[:enabled_actions].split(',')
-          WorkUnit.dequeue(actions, offset)
+          worker, actions = params[:worker_name], params[:worker_actions].split(',')
+          WorkUnit.dequeue(worker, actions, offset)
         end
       end
       

data/lib/cloud_crowd/models/job.rb

@@ -22,7 +22,7 @@ module CloudCrowd
         :inputs       => h['inputs'].to_json,
         :action       => h['action'],
         :options      => (h['options'] || {}).to_json,
-        :owner_email  => h['owner_email'],
+        :email        => h['email'],
         :callback_url => h['callback_url']
       )
     end
@@ -97,11 +97,6 @@ module CloudCrowd
       raise ActionNotFound, "no action named: '#{self.action}' could be found"
     end
     
-    # Get the displayable status name of the Job's status code.
-    def display_status
-      CloudCrowd.display_status(self.status)
-    end
-    
     # How complete is this Job?
     def percent_complete
       return 0   if splitting?
@@ -125,14 +120,15 @@ module CloudCrowd
     # WorkUnits, as well as any completed outputs.
     def to_json(opts={})
       atts = {
-        'id'                => self.id,
-        'color'             => self.color,
-        'status'            => self.display_status, 
-        'percent_complete'  => self.percent_complete,
-        'work_units'        => self.work_units.count,
-        'time_taken'        => self.time_taken
+        'id'                => id,
+        'color'             => color,
+        'status'            => display_status, 
+        'percent_complete'  => percent_complete,
+        'work_units'        => work_units.count,
+        'time_taken'        => time_taken
       }
-      atts.merge!({'outputs' => JSON.parse(self.outputs)}) if self.outputs
+      atts['outputs'] = JSON.parse(outputs) if outputs
+      atts['email']   = email               if email
       atts.to_json
     end
     

data/lib/cloud_crowd/models/work_unit.rb

@@ -8,6 +8,7 @@ module CloudCrowd
     include ModelStatus
     
     belongs_to :job
+    belongs_to :worker_record
     
     validates_presence_of :job_id, :status, :input, :action
     
@@ -17,13 +18,13 @@ module CloudCrowd
     # +enabled_actions+ must be passed to whitelist the types of WorkUnits than
     # can be retrieved for processing. Optionally, specify the +offset+ to peek
     # further on in line.
-    def self.dequeue(enabled_actions=[], offset=0)
+    def self.dequeue(worker_name, enabled_actions=[], offset=0)
       unit = self.first(
-        :conditions => {:status => INCOMPLETE, :taken => false, :action => enabled_actions}, 
+        :conditions => {:status => INCOMPLETE, :worker_record_id => nil, :action => enabled_actions}, 
         :order      => "created_at asc",
         :offset     => offset
       )
-      unit ? unit.update_attributes(:taken => true) && unit : nil
+      unit ? unit.assign_to(worker_name) : nil
     end
     
     # After saving a WorkUnit, its Job should check if it just became complete.
@@ -34,11 +35,11 @@ module CloudCrowd
     # Mark this unit as having finished successfully.
     def finish(output, time_taken)
       update_attributes({
-        :status   => SUCCEEDED,
-        :taken    => false,
-        :attempts => self.attempts + 1,
-        :output   => output,
-        :time     => time_taken
+        :status         => SUCCEEDED,
+        :worker_record  => nil,
+        :attempts       => self.attempts + 1,
+        :output         => output,
+        :time           => time_taken
       })
     end
     
@@ -47,22 +48,29 @@ module CloudCrowd
       tries = self.attempts + 1
       return try_again if tries < CloudCrowd.config[:work_unit_retries]
       update_attributes({
-        :status   => FAILED,
-        :taken    => false,
-        :attempts => tries,
-        :output   => output,
-        :time     => time_taken
+        :status         => FAILED,
+        :worker_record  => nil,
+        :attempts       => tries,
+        :output         => output,
+        :time           => time_taken
       })
     end
     
     # Ever tried. Ever failed. No matter. Try again. Fail again. Fail better.
     def try_again
       update_attributes({
-        :taken    => false,
-        :attempts => self.attempts + 1
+        :worker_record  => nil,
+        :attempts       => self.attempts + 1
       })
     end
     
+    # When a Worker checks out a WorkUnit, establish the connection between
+    # WorkUnit and WorkerRecord.
+    def assign_to(worker_name)
+      self.worker_record = WorkerRecord.find_by_name!(worker_name)
+      self.save ? self : nil
+    end
+    
     # The JSON representation of a WorkUnit shares the Job's options with all
     # its sister WorkUnits.
     def to_json

data/lib/cloud_crowd/models/worker_record.rb

@@ -0,0 +1,61 @@
+module CloudCrowd
+
+  # A WorkerRecord is a recording of an active worker daemon running remotely.
+  # Every time it checks in, we keep track of its status. The attributes shown
+  # here may lag their actual values by up to Worker::CHECK_IN_INTERVAL seconds.
+  class WorkerRecord < ActiveRecord::Base
+        
+    EXPIRES_AFTER = 2 * Worker::CHECK_IN_INTERVAL
+    
+    has_one :work_unit
+    
+    validates_presence_of :name, :thread_status
+    
+    before_destroy :clear_work_units
+    
+    named_scope :alive, lambda { {:conditions => ['updated_at > ?', Time.now - EXPIRES_AFTER]} }
+    named_scope :dead,  lambda { {:conditions => ['updated_at <= ?', Time.now - EXPIRES_AFTER]} }
+    
+    # Save a Worker's current status to the database.
+    def self.check_in(params)
+      attrs = {:thread_status => params[:thread_status], :updated_at => Time.now}
+      self.find_or_create_by_name(params[:name]).update_attributes!(attrs)
+    end
+    
+    # Remove a terminated Worker's record from the database.
+    def self.check_out(params)
+      self.find_by_name(params[:name]).destroy
+    end 
+    
+    # We consider the worker to be alive if it's checked in more recently
+    # than twice the expected interval ago.
+    def alive?
+      updated_at > Time.now - EXPIRES_AFTER
+    end
+    
+    # Derive the Worker's PID on the remote machine from the name.
+    def pid
+      @pid ||= self.name.split('@').first
+    end
+    
+    # Derive the hostname from the Worker's name.
+    def hostname
+      @hostname ||= self.name.split('@').last
+    end
+    
+    def to_json(opts={})
+      {
+        'name'    => name, 
+        'status'  => work_unit && work_unit.display_status,
+      }.to_json
+    end
+    
+    
+    private
+    
+    def clear_work_units
+      WorkUnit.update_all('worker_record_id = null', "worker_record_id = #{id}")
+    end
+    
+  end
+end

data/lib/cloud_crowd/models.rb

@@ -27,8 +27,14 @@ module CloudCrowd
     def complete?;    COMPLETE.include?(self.status);     end
     def incomplete?;  INCOMPLETE.include?(self.status);   end
     
+    # Get the displayable status name of the model's status code.
+    def display_status
+      CloudCrowd.display_status(self.status)
+    end
+    
   end
 end
 
 require 'cloud_crowd/models/job'
-require 'cloud_crowd/models/work_unit'
+require 'cloud_crowd/models/work_unit'
+require 'cloud_crowd/models/worker_record'

data/lib/cloud_crowd/schema.rb

@@ -9,7 +9,7 @@ ActiveRecord::Schema.define(:version => 1) do
     t.text     "outputs"
     t.float    "time"
     t.string   "callback_url"
-    t.string   "owner_email"
+    t.string   "email"
     t.integer  "lock_version", :default => 0, :null => false
     t.datetime "created_at"
     t.datetime "updated_at"
@@ -22,15 +22,24 @@ ActiveRecord::Schema.define(:version => 1) do
     t.string   "action",                          :null => false
     t.integer  "attempts",     :default => 0,     :null => false
     t.integer  "lock_version", :default => 0,     :null => false
-    t.boolean  "taken",        :default => false, :null => false
+    t.integer  "worker_record_id"
     t.float    "time"
     t.text     "output"
     t.datetime "created_at"
     t.datetime "updated_at"
   end
+  
+  create_table "worker_records", :force => true do |t|
+    t.string   "name",          :null => false
+    t.string   "thread_status", :null => false
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
 
   add_index "jobs", ["status"], :name => "index_jobs_on_status"
   add_index "work_units", ["job_id"], :name => "index_work_units_on_job_id"
-  add_index "work_units", ["status", "taken", "action"], :name => "index_work_units_on_status_and_taken_and_action"
+  add_index "work_units", ["status", "worker_record_id", "action"], :name => "index_work_units_on_status_and_worker_record_id_and_action"
+  add_index "worker_records", ["name"], :name => "index_worker_records_on_name"
+  add_index "worker_records", ["updated_at"], :name => "index_worker_records_on_updated_at"
 
 end