cloud-crowd 0.3.3 → 0.4.0

data/public/css/admin_console.css

@@ -22,7 +22,7 @@ body {
     width: 236px; height: 91px;
     background: url(../images/logo.png);
   }
-  
+
 #disconnected {
   position: absolute;
   top: 122px; right: 15px;
@@ -42,70 +42,27 @@ body {
     margin-right: 3px;
   }
 
-#queue {
+#stats {
   position: absolute;
-  top: 16px; left: 327px; right: 15px;
+  top: 16px; left: 327px; right: 22px;
   height: 77px;
   overflow: hidden;
+  color: #454545;
 }
-  #no_jobs {
-    text-align: left;
-    position: absolute;
-    bottom: 8px; right: 8px;
-    color: #999;
-    display: none;
+  #stats tr.data td {
+    padding: 0 50px 0 0;
+    font-size: 50px;
   }
-    #queue.no_jobs #no_jobs {
-      display: block;
+    #stats tr.data td.last {
+      padding-right: 0;
     }
-  #queue_fill {
-    position: absolute;
-    left: 0; right: 0; top: 0;
-    height: 75px;
-    border: 1px solid #5c5c5c;
-    -moz-border-radius: 10px; -webkit-border-radius: 10px; border-radius: 10px;
-    background: transparent url(../images/queue_fill.png) repeat-x 0px -1px;
-  }
-    #queue.no_jobs #queue_fill {
-      opacity: 0.3;
+    #stats tr.data td div {
+      border-bottom: 1px solid #777;
     }
-  #queue .job {
-    position: relative;
-    margin-top: 1px;
-    height: 75px;
-    background: blue;
-    float: left;
-    overflow: hidden;
-    -moz-border-radius: 10px;
-    -webkit-border-radius: 10px;
+  #stats tr.labels td {
+    font-size: 10px;
+    text-transform: uppercase;
   }
-    #queue .completion {
-      position: absolute;
-      bottom: -1px;
-      height: 30px;
-      background: black;
-      border: 1px solid white;
-      -moz-border-radius: 10px; -webkit-border-radius: 10px;
-      opacity: 0.5;
-      overflow: hidden;
-    }
-      #queue .completion.zero {
-        border: 0;
-      }
-    #queue .percent_complete {
-      position: absolute;
-      bottom: 8px; left: 8px;
-      color: #c7c7c7;
-      z-index: 10;
-    }
-    #queue .job_id {
-      color: #333;
-      font-size: 14px;
-      position: absolute;
-      top: 8px; left: 8px;
-      z-index: 10;
-    }
-    
 #sidebar {
   position: absolute;
   top: 120px; left: 10px; bottom: 10px;
@@ -134,11 +91,17 @@ body {
     }
   #sidebar_header {
     position: absolute;
-    width: 250px;
+    width: 283px;
     top: 5px; left: 8px;
     color: #404040;
     text-shadow: 0px 1px 1px #eee;
   }
+    #tail_log {
+      text-transform: none;
+      text-decoration: underline;
+      cursor: pointer;
+      float: right;
+    }
     #sidebar_header.no_nodes .no_nodes,
     #sidebar_header .has_nodes {
       display: block;
@@ -185,7 +148,7 @@ body {
           border-radius: 4px; -moz-border-radius: 4px; -webkit-border-radius: 4px;
           background-color: #ccc;
         }
-      
+
 #worker_info {
   position: absolute;
   width: 231px; height: 79px;
@@ -204,15 +167,15 @@ body {
       background: url(../images/worker_info_loading.gif) no-repeat right bottom;
       width: 45px; height: 9px;
     }
-    #worker_info.awake #worker_details, 
+    #worker_info.awake #worker_details,
     #worker_sleeping {
       display: block;
     }
-    #worker_details, #worker_info.loading #worker_details, 
+    #worker_details, #worker_info.loading #worker_details,
     #worker_info.loading #worker_sleeping, #worker_info.awake #worker_sleeping {
       display: none;
     }
-      
+
 #graphs {
   position: absolute;
   padding: 17px 15px 15px 17px;

data/public/js/admin_console.js

@@ -4,22 +4,22 @@
 // Think about pulling in the DCJS framework, instead of just raw jQuery here.
 // Leaving it hacked together like this just cries out for templates, dunnit?
 window.Console = {
-  
+
   // Maximum number of data points to record and graph.
   MAX_DATA_POINTS : 100,
-  
+
   // Milliseconds between polling the central server for updates to Job progress.
-  POLL_INTERVAL : 3000,
-  
+  POLL_INTERVAL : 6000,
+
   // Default speed for all animations.
   ANIMATION_SPEED : 300,
-  
+
   // Keep this in sync with the map in cloud-crowd.rb
-  DISPLAY_STATUS_MAP : ['unknown', 'processing', 'succeeded', 'failed', 'splitting', 'merging'],    
-  
+  DISPLAY_STATUS_MAP : ['unknown', 'processing', 'succeeded', 'failed', 'splitting', 'merging'],
+
   // Images to preload
   PRELOAD_IMAGES : ['images/server_error.png'],
-  
+
   // All options for drawing the system graphs.
   GRAPH_OPTIONS : {
     xaxis   : {mode : 'time', timeformat : '%M:%S'},
@@ -31,37 +31,40 @@ window.Console = {
   NODES_COLOR       : '#1870ab',
   WORKERS_COLOR     : '#45a4e5',
   WORK_UNITS_COLOR  : '#ffba14',
-  
+
   // Starting the console begins polling the server.
   initialize : function() {
-    this._jobsHistory = [];
-    this._nodesHistory = [];
-    this._workersHistory = [];
-    this._workUnitsHistory = [];
-    this._histories = [this._jobsHistory, this._nodesHistory, this._workersHistory, this._workUnitsHistory];
-    this._queue = $('#jobs');
-    this._workerInfo = $('#worker_info');
+    this._jobsHistory       = [];
+    this._nodesHistory      = [];
+    this._workersHistory    = [];
+    this._workUnitsHistory  = [];
+    this._histories         = [this._jobsHistory, this._nodesHistory, this._workersHistory, this._workUnitsHistory];
+    this._workerInfo        = $('#worker_info');
+    this._jobCountEl        = $('#job_count');
+    this._workUnitCountEl   = $('#work_unit_count');
+    this._nodeCountEl       = $('#node_count');
+    this._workerCountEl     = $('#worker_count');
     this._disconnected = $('#disconnected');
     $(window).bind('resize', Console.renderGraphs);
     $('#nodes .worker').live('click', Console.getWorkerInfo);
+    $('#tail_log').bind('click', Console.tailLog);
     $('#workers_legend').css({background : this.WORKERS_COLOR});
     $('#nodes_legend').css({background : this.NODES_COLOR});
     this.getStatus();
     $.each(this.PRELOAD_IMAGES, function(){ var i = new Image(); i.src = this; });
   },
-  
+
   // Request the lastest status of all jobs and workers, re-render or update
   // the DOM to reflect.
   getStatus : function() {
     $.ajax({url : 'status', dataType : 'json', success : function(resp) {
-      Console._jobs           = resp.jobs;
+      Console._jobCount       = resp.job_count;
       Console._nodes          = resp.nodes;
       Console._workUnitCount  = resp.work_unit_count;
       Console._workerCount    = Console.countWorkers();
       Console.recordDataPoint();
       if (Console._disconnected.is(':visible')) Console._disconnected.fadeOut(Console.ANIMATION_SPEED);
-      $('#queue').toggleClass('no_jobs', Console._jobs.length <= 0);
-      Console.renderJobs();
+      Console.renderStats();
       Console.renderNodes();
       Console.renderGraphs();
       setTimeout(Console.getStatus, Console.POLL_INTERVAL);
@@ -70,52 +73,32 @@ window.Console = {
       setTimeout(Console.getStatus, Console.POLL_INTERVAL);
     }});
   },
-  
+
+  // Fetch the last 100 lines of log from the server.
+  tailLog : function() {
+    $.ajax({url : 'log', success : function(resp) {
+      var win = window.open('');
+      win.document.open();
+      win.document.write('<pre>' + resp + '</pre>');
+      win.document.close();
+    }});
+  },
+
   // Count the total number of workers in the current list of nodes.
   countWorkers : function() {
     var sum = 0;
     for (var i=0; i < this._nodes.length; i++) sum += this._nodes[i].workers.length;
     return sum;
   },
-  
-  // Render an individual job afresh.
-  renderJob : function(job) {
-    this._queue.append('<div class="job" id="job_' + job.id + '" style="width:' + job.width + '%; background: #' + job.color + ';"><div class="completion ' + (job.percent_complete <= 0 ? 'zero' : '') + '" style="width:' + job.percent_complete + '%;"></div><div class="percent_complete">' + job.percent_complete + '%</div><div class="job_id">#' + job.id + '</div></div>');
-  },
-  
-  // Animate the update to an existing job in the queue.
-  updateJob : function(job, jobEl) {
-    jobEl.animate({width : job.width + '%'}, this.ANIMATION_SPEED);
-    var completion = $('.completion', jobEl);
-    if (job.percent_complete > 0) completion.removeClass('zero');
-    completion.animate({width : job.percent_complete + '%'}, this.ANIMATION_SPEED);
-    $('.percent_complete', jobEl).html(job.percent_complete + '%');
-  },
-  
-  // Render all jobs, calculating relative widths and completions.
-  renderJobs : function() {
-    var totalUnits = 0;
-    var totalWidth = this._queue.width();
-    var jobIds = [];
-    $.each(this._jobs, function() { 
-      jobIds.push(this.id);
-      totalUnits += this.work_units; 
-    });
-    $.each($('.job'), function() {
-      var el = this;
-      if (jobIds.indexOf(parseInt(el.id.replace(/\D/g, ''), 10)) < 0) {
-        $(el).animate({width : '0%'}, Console.ANIMATION_SPEED - 50, 'linear', function() {
-          $(el).remove();
-        });
-      }
-    });
-    $.each(this._jobs, function() {
-      this.width  = (this.work_units / totalUnits) * 100;
-      var jobEl = $('#job_' + this.id);
-      jobEl[0] ? Console.updateJob(this, jobEl) : Console.renderJob(this);
-    });
+
+  // Render the numeric statistic counts.
+  renderStats : function() {
+    this._jobCountEl.text(this._jobCount);
+    this._workUnitCountEl.text(this._workUnitCount);
+    this._nodeCountEl.text(this._nodes.length);
+    this._workerCountEl.text(this._workerCount);
   },
-  
+
   // Re-render all workers from scratch each time.
   // This method is desperately in need of Javascript templates...
   renderNodes : function() {
@@ -123,7 +106,7 @@ window.Console = {
     var nc = this._nodes.length, wc = this._workerCount;
     $('.has_nodes', header).html(nc + " Node" + (nc != 1 ? 's' : '') + " / " + wc + " Worker" + (wc != 1 ? 's' : ''));
     header.toggleClass('no_nodes', this._nodes.length <= 0);
-    $('#nodes').html($.map(this._nodes, function(node) { 
+    $('#nodes').html($.map(this._nodes, function(node) {
       var html = "";
       var extra = node.status == 'busy' ? ' <span class="busy">[busy]</span>' : '';
       html += '<div class="node ' + node.status + '">' + node.host + extra + '</div>';
@@ -134,19 +117,19 @@ window.Console = {
       return html;
     }).join(''));
   },
-  
+
   // Record the current state and re-render all graphs.
   recordDataPoint : function() {
     var timestamp = (new Date()).getTime();
-    this._jobsHistory.push([timestamp, this._jobs.length]);
+    this._jobsHistory.push([timestamp, this._jobCount]);
     this._nodesHistory.push([timestamp, this._nodes.length]);
     this._workersHistory.push([timestamp, this._workerCount]);
     this._workUnitsHistory.push([timestamp, this._workUnitCount]);
-    $.each(this._histories, function() { 
-      if (this.length > Console.MAX_DATA_POINTS) this.shift(); 
+    $.each(this._histories, function() {
+      if (this.length > Console.MAX_DATA_POINTS) this.shift();
     });
   },
-  
+
   // Convert our recorded data points into a format Flot can understand.
   renderGraphs : function() {
     $.plot($('#work_units_graph'), [
@@ -160,7 +143,7 @@ window.Console = {
       {label : 'Workers', color : Console.WORKERS_COLOR, data : Console._workersHistory}
     ], Console.GRAPH_OPTIONS);
   },
-  
+
   // Request the Worker info from the central server.
   getWorkerInfo : function(e) {
     e.stopImmediatePropagation();
@@ -173,7 +156,7 @@ window.Console = {
     $(document).bind('click', Console.hideWorkerInfo);
     return false;
   },
-  
+
   // When we receieve worker info, update the bubble.
   renderWorkerInfo : function(resp) {
     var info = Console._workerInfo;
@@ -185,13 +168,13 @@ window.Console = {
     $('.job_id', info).html(resp.job_id);
     $('.work_unit_id', info).html(resp.id);
   },
-  
+
   // Hide worker info and unbind the global hide handler.
   hideWorkerInfo : function() {
     $(document).unbind('click', Console.hideWorkerInfo);
-    Console._workerInfo.fadeOut(Console.ANIMATION_SPEED); 
+    Console._workerInfo.fadeOut(Console.ANIMATION_SPEED);
   }
-  
+
 };
 
 $(document).ready(function() { Console.initialize(); });

data/test/acceptance/test_server.rb

@@ -1,43 +1,41 @@
 require 'test_helper'
 
 class ServerTest < Test::Unit::TestCase
-  
+
   include Rack::Test::Methods
-  
+
   def app
     CloudCrowd::Server
   end
-  
+
   context "The CloudCrowd::Server (Sinatra)" do
-        
+
     setup do
       Job.destroy_all
       2.times { Job.make }
     end
-    
+
     should "set the identity of the Ruby instance" do
       app.new
       assert CloudCrowd.server?
     end
-    
+
     should "be able to render the Operations Center (GET /)" do
       get '/'
       assert last_response.body.include? '<div id="nodes">'
       assert last_response.body.include? '<div id="graphs">'
     end
-    
+
     should "be able to get the current status for all jobs (GET /status)" do
       resp = JSON.parse(get('/status').body)
-      assert resp['jobs'].length == 2
-      assert resp['jobs'][0]['status'] == 'processing'
-      assert resp['jobs'][0]['percent_complete'] == 0
+      assert resp['job_count'] == 2
       assert resp['work_unit_count'] == 2
     end
-    
+
     should "have a heartbeat" do
       assert get('/heartbeat').body == 'buh-bump'
     end
-    
+
     should "be able to create a job" do
       WorkUnit.expects(:distribute_to_nodes).returns(true)
       post('/jobs', :job => '{"action":"graphics_magick","inputs":["http://www.google.com/"]}')
@@ -47,20 +45,20 @@ class ServerTest < Test::Unit::TestCase
       assert job_info['work_units'] == 1
       assert Job.last.id == job_info['id']
     end
-    
+
     should "be able to check in on the status of a job" do
       get("/jobs/#{Job.last.id}")
       assert last_response.ok?
       assert JSON.parse(last_response.body)['percent_complete'] == 0
     end
-    
+
     should "be able to clean up a job when we're done with it" do
       id = Job.last.id
       delete("/jobs/#{id}")
       assert last_response.successful? && last_response.empty?
       assert !Job.find_by_id(id)
     end
-  
+
   end
-  
+
 end

data/test/unit/test_action.rb

@@ -8,63 +8,65 @@ class EmptyAction < CloudCrowd::Action
 end
 
 class ActionTest < Test::Unit::TestCase
-  
+
   context "A CloudCrowd::Action" do
-        
+
     setup do
       @store = CloudCrowd::AssetStore.new
       @args = [CloudCrowd::PROCESSING, 'file://' + File.expand_path(__FILE__), {'job_id' => 1, 'work_unit_id' => 1}, @store]
       @action = CloudCrowd.actions['word_count'].new(*@args)
     end
-    
+
     should "throw an exception if the 'process' method isn't implemented" do
       assert_raise(NotImplementedError) { EmptyAction.new(*@args).process }
     end
-    
+
     should "have downloaded the input URL to local storage" do
       assert @action.input_path
       assert File.read(@action.input_path) == File.read(File.expand_path(__FILE__))
     end
-    
+
     should "be able to save (to the filesystem while testing)" do
       assert @action.save(@action.input_path) == "file://#{@store.local_storage_path}/word_count/job_1/unit_1/test_action.rb"
     end
-    
+
     should "be able to clean up after itself" do
       @action.cleanup_work_directory
       assert !File.exists?(@action.work_directory)
     end
-    
+
     should "be able to generate a safe filename for a URL to write to disk" do
       name = @action.safe_filename("http://example.com/Some%20(Crazy'Kinda%7E)'Filename.txt")
       assert name == 'Some-Crazy-Kinda-Filename.txt'
+      name = @action.safe_filename("http://example.com/file.pdf?one=two&three=four")
+      assert name == 'file.pdf'
     end
-    
+
     should "be able to count the number of words in this file" do
-      assert @action.process == 212
+      assert @action.process == 219
     end
-    
+
     should "raise an exception when backticks fail" do
       def @action.process; `utter failure 2>&1`; end
       assert_raise(CloudCrowd::Error::CommandFailed) { @action.process }
     end
-            
+
   end
 
 
   context "A CloudCrowd::Action without URL input" do
-        
+
     setup do
       @store = CloudCrowd::AssetStore.new
       @args = [CloudCrowd::PROCESSING, 'inputstring', {'job_id' => 1, 'work_unit_id' => 1}, @store]
       @action = CloudCrowd.actions['word_count'].new(*@args)
     end
-    
+
     should "should not interpret the input data as an url" do
       assert_equal 'inputstring', @action.input
       assert_nil @action.input_path
     end
-    
+
   end
-  
+
 end