cloud-crowd 0.1.0 → 0.2.0

data/lib/cloud_crowd/worker.rb

@@ -1,90 +1,48 @@
 module CloudCrowd
   
-  # The Worker, run at intervals by the Daemon, fetches WorkUnits from the
-  # central server and dispatches Actions to process them. Workers only fetch
-  # units that they are able to handle (for which they have an action in their
-  # actions directory). If communication with the central server is interrupted, 
-  # the WorkUnit will repeatedly attempt to complete its unit -- every 
-  # Worker::RETRY_WAIT seconds. Any exceptions that take place during 
+  # The Worker, forked off from the Node when a new WorkUnit is received, 
+  # launches an Action for processing. Workers will only ever receive WorkUnits
+  # that they are able to handle (for which they have a corresponding action in 
+  # their actions directory). If communication with the central server is 
+  # interrupted, the Worker will repeatedly attempt to complete its unit -- 
+  # every Worker::RETRY_WAIT seconds. Any exceptions that take place during 
   # the course of the Action will cause the Worker to mark the WorkUnit as 
-  # having failed.
+  # having failed. When finished, the Worker's process exits, minimizing the
+  # potential for memory leaks.
   class Worker
     
-    # The time between worker check-ins with the central server, informing
-    # it of the current status, and simply that it's still alive.
-    CHECK_IN_INTERVAL = 60
-    
     # Wait five seconds to retry, after internal communcication errors.
     RETRY_WAIT = 5
             
     attr_reader :action
     
-    # Spinning up a worker will create a new AssetStore with a persistent
-    # connection to S3. This AssetStore gets passed into each action, for use
-    # as it is run.
-    def initialize
-      @id               = $$
-      @hostname         = Socket.gethostname
-      @name             = "#{@id}@#{@hostname}"
-      @store            = AssetStore.new
-      @server           = CloudCrowd.central_server
-      @enabled_actions  = CloudCrowd.actions.keys
-      log 'started'
+    # A new Worker begins processing its WorkUnit straight off.
+    def initialize(node, work_unit)
+      @pid  = $$
+      @node = node
+      trap_signals
+      setup_work_unit(work_unit)
+      run
     end
     
-    # Ask the central server for the first WorkUnit in line.
-    def fetch_work_unit
-      keep_trying_to "fetch a new work unit" do
-        unit_json = @server['/work'].post(base_params)
-        setup_work_unit(unit_json)
-      end
-    end
-    
-    # Return output to the central server, marking the current work unit as done.
+    # Return output to the central server, marking the WorkUnit done.
     def complete_work_unit(result)
       keep_trying_to "complete work unit" do
-        data = completion_params.merge({:status => 'succeeded', :output => result})
-        unit_json = @server["/work/#{data[:id]}"].put(data)
+        data = base_params.merge({:status => 'succeeded', :output => result})
+        @node.server["/work/#{data[:id]}"].put(data)
         log "finished #{display_work_unit} in #{data[:time]} seconds"
-        clear_work_unit
-        setup_work_unit(unit_json)
       end
     end
     
-    # Mark the current work unit as failed, returning the exception to central.
+    # Mark the WorkUnit failed, returning the exception to central.
     def fail_work_unit(exception)
       keep_trying_to "mark work unit as failed" do
-        data = completion_params.merge({:status => 'failed', :output => {'output' => exception.message}.to_json})
-        unit_json = @server["/work/#{data[:id]}"].put(data)
+        data = base_params.merge({:status => 'failed', :output => {'output' => exception.message}.to_json})
+        @node.server["/work/#{data[:id]}"].put(data)
         log "failed #{display_work_unit} in #{data[:time]} seconds\n#{exception.message}\n#{exception.backtrace}"
-        clear_work_unit
-        setup_work_unit(unit_json)
-      end
-    end
-    
-    # Check in with the central server. Let it know the condition of the work 
-    # thread, the action and status we're processing, and our hostname and PID.
-    def check_in(thread_status)
-      keep_trying_to "check in with central" do
-        @server["/worker"].put({
-          :name          => @name,
-          :thread_status => thread_status
-        })
       end
     end
     
-    # Inform the central server that this worker is finished. This is the only
-    # remote method that doesn't retry on connection errors -- if the worker 
-    # can't connect to the central server while it's trying to shutdown, it 
-    # should close, regardless.
-    def check_out
-      @server["/worker"].put({
-        :name       => @name,
-        :terminated => true
-      })
-      log 'exiting'
-    end
-    
     # We expect and require internal communication between the central server
     # and the workers to succeed. If it fails for any reason, log it, and then 
     # keep trying the same request.
@@ -100,36 +58,38 @@ module CloudCrowd
       end
     end
     
-    # Does this Worker have a job to do?
-    def has_work?
-      @action_name && @input && @options
-    end
-    
-    # Loggable string of the current work unit.
+    # Loggable details describing what the Worker is up to.
     def display_work_unit
-      "unit ##{@options['work_unit_id']} (#{@action_name})"
+      "unit ##{@options['work_unit_id']} (#{@action_name}/#{CloudCrowd.display_status(@status)})"
     end
     
-    # Executes the current work unit, catching all exceptions as failures.
+    # Executes the WorkUnit by running the Action, catching all exceptions as 
+    # failures. We capture the thread so that we can kill it from the outside,
+    # when exiting.
     def run_work_unit
-      begin
-        result = nil
-        @action = CloudCrowd.actions[@action_name].new(@status, @input, @options, @store)
-        Dir.chdir(@action.work_directory) do
-          result = case @status
-          when PROCESSING then @action.process
-          when SPLITTING  then @action.split
-          when MERGING    then @action.merge
-          else raise Error::StatusUnspecified, "work units must specify their status"
+      @worker_thread = Thread.new do
+        begin
+          result = nil
+          @action = CloudCrowd.actions[@action_name].new(@status, @input, @options, @node.asset_store)
+          Dir.chdir(@action.work_directory) do
+            result = case @status
+            when PROCESSING then @action.process
+            when SPLITTING  then @action.split
+            when MERGING    then @action.merge
+            else raise Error::StatusUnspecified, "work units must specify their status"
+            end
           end
+          complete_work_unit({'output' => result}.to_json)
+        rescue Exception => e
+          fail_work_unit(e)
+        ensure
+          @action.cleanup_work_directory
         end
-        complete_work_unit({'output' => result}.to_json)
-      rescue Exception => e
-        fail_work_unit(e)
       end
+      @worker_thread.join
     end
     
-    # Wraps <tt>run_work_unit</tt> to benchmark the execution time, if requested.
+    # Wraps run_work_unit to benchmark the execution time, if requested.
     def run
       return run_work_unit unless @options['benchmark']
       status = CloudCrowd.display_status(@status)
@@ -139,27 +99,17 @@ module CloudCrowd
     
     private
     
-    # Common parameters to send back to central.
-    def base_params
-      @base_params ||= {
-        :worker_name    => @name, 
-        :worker_actions => @enabled_actions.join(',')
-      }
-    end
-    
     # Common parameters to send back to central upon unit completion, 
     # regardless of success or failure.
-    def completion_params
-      base_params.merge({
-        :id       => @options['work_unit_id'], 
-        :time     => Time.now - @start_time 
-      })
+    def base_params
+      { :pid  => @pid,
+        :id   => @options['work_unit_id'], 
+        :time => Time.now - @start_time }
     end
     
-    # Extract our instance variables from a WorkUnit's JSON.
-    def setup_work_unit(unit_json)
-      return false unless unit_json
-      unit = JSON.parse(unit_json)
+    # Extract the Worker's instance variables from a WorkUnit's JSON.
+    def setup_work_unit(unit)
+      return false unless unit
       @start_time = Time.now
       @action_name, @input, @options, @status = unit['action'], unit['input'], unit['options'], unit['status']
       @options['job_id'] = unit['job_id']
@@ -171,14 +121,25 @@ module CloudCrowd
     
     # Log a message to the daemon log. Includes PID for identification.
     def log(message)
-      puts "Worker ##{@id}: #{message}" unless ENV['RACK_ENV'] == 'test'
+      puts "Worker ##{@pid}: #{message}" unless ENV['RACK_ENV'] == 'test'
     end
     
-    # When we're done with a unit, clear out our instance variables to make way 
-    # for the next one. Also, remove all of the unit's temporary storage.
-    def clear_work_unit
-      @action.cleanup_work_directory
-      @action, @action_name, @input, @options, @start_time = nil, nil, nil, nil, nil
+    # When signaled to exit, make sure that the Worker shuts down cleanly.
+    def trap_signals
+      Signal.trap('INT') { shut_down }
+      Signal.trap('KILL') { shut_down }
+      Signal.trap('TERM') { shut_down }
+    end
+    
+    # Force the Worker to quit, even if it's in the middle of processing.
+    # If it had a checked-out WorkUnit, the Node should have released it on
+    # the central server already.
+    def shut_down
+      if @worker_thread
+        @worker_thread.kill
+        @worker_thread.kill! if @worker_thread.alive?
+      end
+      Process.exit
     end
     
   end

data/public/css/admin_console.css

@@ -134,44 +134,57 @@ body {
     }
   #sidebar_header {
     position: absolute;
+    width: 250px;
     top: 5px; left: 8px;
     color: #404040;
     text-shadow: 0px 1px 1px #eee;
   }
-    #sidebar_header.no_workers .no_workers,
-    #sidebar_header .has_workers {
+    #sidebar_header.no_nodes .no_nodes,
+    #sidebar_header .has_nodes {
       display: block;
     }
-    #sidebar_header .no_workers,
-    #sidebar_header.no_workers .has_workers {
+    #sidebar_header .no_nodes,
+    #sidebar_header.no_nodes .has_nodes {
       display: none;
     }
-  #workers {
+  #nodes {
     position: absolute;
     padding: 2px 0;
     top: 21px; left: 0; bottom: 21px;
     width: 298px;
     overflow-y: auto; overflow-x: hidden;
   }
-    #workers .worker {
+    #nodes .node, #nodes .worker {
       border: 1px solid transparent;
       margin: 1px 7px;
       padding-left: 18px;
-      font-size: 11px;
-      line-height: 22px;
-      background: url(/images/bullet_white.png) no-repeat left center;
-      cursor: pointer;
+      background-position: left center;
+      background-repeat: no-repeat;
     }
-      #workers .worker.processing,
-      #workers .worker.splitting,
-      #workers .worker.merging {
-        background: url(/images/bullet_green.png) no-repeat left center;
+      #nodes .node {
+        font-size: 11px;
+        line-height: 22px;
+        background-image: url(/images/server.png);
       }
-      #workers .worker:hover {
-        border: 1px solid #aaa;
-        border-radius: 4px; -moz-border-radius: 4px; -webkit-border-radius: 4px;
-        background-color: #ccc;
+        #nodes .node.busy {
+          background-image: url(/images/server_busy.png);
+        }
+          #nodes .node.busy span.busy {
+            font-size: 9px;
+            color: #7f7f7f;
+            text-transform: uppercase;
+          }
+      #nodes .worker {
+        font-size: 10px;
+        line-height: 18px;
+        cursor: pointer;
+        background-image: url(/images/bullet_green.png);
       }
+        #nodes .worker:hover {
+          border: 1px solid #aaa;
+          border-radius: 4px; -moz-border-radius: 4px; -webkit-border-radius: 4px;
+          background-color: #ccc;
+        }
       
 #worker_info {
   position: absolute;
@@ -216,6 +229,15 @@ body {
       text-shadow: 0px 1px 1px #eee;
       margin-bottom: 10px;
     }
+      .legend_box {
+        display: inline-block;
+        width: 10px; height: 10px;
+        border: 1px solid #bbb;
+        position: relative;
+        top: 1px;
+        margin: 0 1px;
+        background-color: #a1003d;
+      }
     .graph {
       height: 150px;
     }

data/public/js/admin_console.js

@@ -22,26 +22,30 @@ window.Console = {
   
   // All options for drawing the system graphs.
   GRAPH_OPTIONS : {
-    xaxis :   {mode : 'time', timeformat : '%M:%S'},
-    yaxis :   {tickDecimals : 0},
-    legend :  {show : false},
-    grid :    {backgroundColor : '#7f7f7f', color : '#555', tickColor : '#666', borderWidth : 2}
+    xaxis   : {mode : 'time', timeformat : '%M:%S'},
+    yaxis   : {tickDecimals : 0},
+    legend  : {show : false},
+    grid    : {backgroundColor : '#7f7f7f', color : '#555', tickColor : '#666', borderWidth : 2}
   },
   JOBS_COLOR        : '#db3a0f',
-  WORKERS_COLOR     : '#a1003d',
+  NODES_COLOR       : '#1870ab',
+  WORKERS_COLOR     : '#45a4e5',
   WORK_UNITS_COLOR  : '#ffba14',
   
   // Starting the console begins polling the server.
   initialize : function() {
     this._jobsHistory = [];
+    this._nodesHistory = [];
     this._workersHistory = [];
     this._workUnitsHistory = [];
-    this._histories = [this._jobsHistory, this._workersHistory, this._workUnitsHistory];
+    this._histories = [this._jobsHistory, this._nodesHistory, this._workersHistory, this._workUnitsHistory];
     this._queue = $('#jobs');
     this._workerInfo = $('#worker_info');
     this._disconnected = $('#disconnected');
     $(window).bind('resize', Console.renderGraphs);
-    $('#workers .worker').live('click', Console.getWorkerInfo);
+    $('#nodes .worker').live('click', Console.getWorkerInfo);
+    $('#workers_legend').css({background : this.WORKERS_COLOR});
+    $('#nodes_legend').css({background : this.NODES_COLOR});
     this.getStatus();
     $.each(this.PRELOAD_IMAGES, function(){ var i = new Image(); i.src = this; });
   },
@@ -51,13 +55,14 @@ window.Console = {
   getStatus : function() {
     $.ajax({url : '/status', dataType : 'json', success : function(resp) {
       Console._jobs           = resp.jobs;
-      Console._workers        = resp.workers;
+      Console._nodes          = resp.nodes;
       Console._workUnitCount  = resp.work_unit_count;
+      Console._workerCount    = Console.countWorkers();
       Console.recordDataPoint();
       if (Console._disconnected.is(':visible')) Console._disconnected.fadeOut(Console.ANIMATION_SPEED);
       $('#queue').toggleClass('no_jobs', Console._jobs.length <= 0);
       Console.renderJobs();
-      Console.renderWorkers();
+      Console.renderNodes();
       Console.renderGraphs();
       setTimeout(Console.getStatus, Console.POLL_INTERVAL);
     }, error : function(request, status, errorThrown) {
@@ -66,6 +71,13 @@ window.Console = {
     }});
   },
   
+  // Count the total number of workers in the current list of nodes.
+  countWorkers : function() {
+    var sum = 0;
+    for (var i=0; i < this._nodes.length; i++) sum += this._nodes[i].workers.length;
+    return sum;
+  },
+  
   // Render an individual job afresh.
   renderJob : function(job) {
     this._queue.append('<div class="job" id="job_' + job.id + '" style="width:' + job.width + '%; background: #' + job.color + ';"><div class="completion ' + (job.percent_complete <= 0 ? 'zero' : '') + '" style="width:' + job.percent_complete + '%;"></div><div class="percent_complete">' + job.percent_complete + '%</div><div class="job_id">#' + job.id + '</div></div>');
@@ -105,12 +117,21 @@ window.Console = {
   },
   
   // Re-render all workers from scratch each time.
-  renderWorkers : function() {
+  // This method is desperately in need of Javascript templates...
+  renderNodes : function() {
     var header = $('#sidebar_header');
-    $('.has_workers', header).html(this._workers.length + " Active Worker Daemons");
-    header.toggleClass('no_workers', this._workers.length <= 0);
-    $('#workers').html($.map(this._workers, function(w) { 
-      return '<div class="worker ' + w.status + '" rel="' + w.name + '">' + w.name + '</div>';
+    var nc = this._nodes.length, wc = this._workerCount;
+    $('.has_nodes', header).html(nc + " Node" + (nc != 1 ? 's' : '') + " / " + wc + " Worker" + (wc != 1 ? 's' : ''));
+    header.toggleClass('no_nodes', this._nodes.length <= 0);
+    $('#nodes').html($.map(this._nodes, function(node) { 
+      var html = "";
+      var extra = node.status == 'busy' ? ' <span class="busy">[busy]</span>' : '';
+      html += '<div class="node ' + node.status + '">' + node.host + extra + '</div>';
+      html += $.map(node.workers, function(pid) {
+        var name = pid + '@' + node.host;
+        return '<div class="worker" rel="' + name + '">' + name + '</div>';
+      }).join('');
+      return html;
     }).join(''));
   },
   
@@ -118,7 +139,8 @@ window.Console = {
   recordDataPoint : function() {
     var timestamp = (new Date()).getTime();
     this._jobsHistory.push([timestamp, this._jobs.length]);
-    this._workersHistory.push([timestamp, this._workers.length]);
+    this._nodesHistory.push([timestamp, this._nodes.length]);
+    this._workersHistory.push([timestamp, this._workerCount]);
     this._workUnitsHistory.push([timestamp, this._workUnitCount]);
     $.each(this._histories, function() { 
       if (this.length > Console.MAX_DATA_POINTS) this.shift(); 
@@ -127,9 +149,16 @@ window.Console = {
   
   // Convert our recorded data points into a format Flot can understand.
   renderGraphs : function() {
-    $.plot($('#jobs_graph'), [{label : 'Jobs in Queue', color : Console.JOBS_COLOR, data : Console._jobsHistory}], Console.GRAPH_OPTIONS);
-    $.plot($('#workers_graph'), [{label : 'Active Workers', color : Console.WORKERS_COLOR, data : Console._workersHistory}], Console.GRAPH_OPTIONS);
-    $.plot($('#work_units_graph'), [{label : 'Work Units in Queue', color : Console.WORK_UNITS_COLOR, data : Console._workUnitsHistory}], Console.GRAPH_OPTIONS);
+    $.plot($('#work_units_graph'), [
+      {label : 'Work Units in Queue', color : Console.WORK_UNITS_COLOR, data : Console._workUnitsHistory}
+    ], Console.GRAPH_OPTIONS);
+    $.plot($('#jobs_graph'), [
+      {label : 'Jobs in Queue', color : Console.JOBS_COLOR, data : Console._jobsHistory}
+    ], Console.GRAPH_OPTIONS);
+    $.plot($('#workers_graph'), [
+      {label : 'Nodes', color : Console.NODES_COLOR, data : Console._nodesHistory},
+      {label : 'Workers', color : Console.WORKERS_COLOR, data : Console._workersHistory}
+    ], Console.GRAPH_OPTIONS);
   },
   
   // Request the Worker info from the central server.