cloud-crowd 0.7.5 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 82aa3ec5f4de9712060de2ba1f62207cc4c9d57c
-  data.tar.gz: 8129d294659a958f2ae1031dba8f4b507d064d03
+  metadata.gz: 1d33763aebe17845688903d5e4c15f7c9be07833
+  data.tar.gz: 95c586ec36d78a80c4e648017ad3e8c245b73206
 SHA512:
-  metadata.gz: 45c9bf5f14369c0b9d30d8eb2e6cc3dcbf6bcf046ccff6e6262682ba94b5bb26b3d23fd4fe4c8c1e7f7a980146abad9f12d5fa3baeeace408e6f2e6ca9db0c38
-  data.tar.gz: a4c1687bf3c7ada3354c30a19eaa48d8709f1c2a9bd3b874e4d823519ee3201600d9f20607d003dc7430c188395474d0ffe52d0e36e900e8f2c1c530b96b1111
+  metadata.gz: f04bca0a15c63970099d8595323237962764a6050d0a433ac4c1b8cb6c00f2881578cd24a769adaf20952ba852f3cb21b2cec667456dcce670442acd05ff0a89
+  data.tar.gz: 4a26778718399824ffbc749c099808b38433d3b6864fb1439336b5bf4d8b1a4c1b50db29c2dc11263c1d11d8c3a5ce105c0dac93bb0134ef76b2626aecdbaf74

data/config/database.example.yml

@@ -2,12 +2,11 @@
 # to use any database that ActiveRecord supports. Only the central server needs
 # this file to be configured -- nodes never connect directly to the database.
 
-:adapter:  mysql
-:encoding: utf8
-:username: root
-:password:
-:socket:   /tmp/mysql.sock
+:adapter:  postgresql
+:encoding: unicode
+:host:   localhost
 :database: cloud_crowd
+allow_concurrency: true
 
 # If you'd prefer to use an SQLite database instead, the following configuration
 # will do nicely:

data/lib/cloud-crowd.rb

@@ -35,21 +35,23 @@ ActiveModel::Serializer.root = false
 module CloudCrowd
 
   # Autoload all the CloudCrowd internals.
-  autoload :Action,       'cloud_crowd/action'
-  autoload :AssetStore,   'cloud_crowd/asset_store'
-  autoload :CommandLine,  'cloud_crowd/command_line'
-  autoload :Helpers,      'cloud_crowd/helpers'
-  autoload :Inflector,    'cloud_crowd/inflector'
-  autoload :Job,          'cloud_crowd/models'
-  autoload :Node,         'cloud_crowd/node'
-  autoload :NodeRecord,   'cloud_crowd/models'
-  autoload :Server,       'cloud_crowd/server'
-  autoload :Worker,       'cloud_crowd/worker'
-  autoload :WorkUnit,     'cloud_crowd/models'
-  autoload :Dispatcher,   'cloud_crowd/dispatcher'
+  autoload :Action,             'cloud_crowd/action'
+  autoload :AssetStore,         'cloud_crowd/asset_store'
+  autoload :CommandLine,        'cloud_crowd/command_line'
+  autoload :Helpers,            'cloud_crowd/helpers'
+  autoload :Inflector,          'cloud_crowd/inflector'
+  autoload :MODELS,             'cloud_crowd/models'
+  autoload :Job,                'cloud_crowd/models'
+  autoload :BlackListedAction,  'cloud_crowd/models'
+  autoload :Node,               'cloud_crowd/node'
+  autoload :NodeRecord,         'cloud_crowd/models'
+  autoload :Server,             'cloud_crowd/server'
+  autoload :Worker,             'cloud_crowd/worker'
+  autoload :WorkUnit,           'cloud_crowd/models'
+  autoload :Dispatcher,         'cloud_crowd/dispatcher'
 
   # Increment the schema version when there's a backwards incompatible change.
-  SCHEMA_VERSION = 4
+  SCHEMA_VERSION = 5
 
   # Root directory of the CloudCrowd gem.
   ROOT           = File.expand_path(File.dirname(__FILE__) + '/..')

data/lib/cloud_crowd/models.rb

@@ -38,3 +38,8 @@ end
 require 'cloud_crowd/models/job'
 require 'cloud_crowd/models/node_record'
 require 'cloud_crowd/models/work_unit'
+require 'cloud_crowd/models/black_listed_action'
+
+module CloudCrowd
+  MODELS = [Job, NodeRecord, WorkUnit, BlackListedAction]
+end

data/lib/cloud_crowd/models/black_listed_action.rb

@@ -0,0 +1,27 @@
+module CloudCrowd
+
+  # A Black Listed Action is an action that has been disabled from running.  For example, we may
+  # want to disable calls to a particular API once we have reached a rate limit.
+  # When a Node exits, it destroys this record.
+  class BlackListedAction < ActiveRecord::Base
+
+    validates_presence_of :action
+    validates :action, uniqueness: true
+    validates :action, format: { with: /\A[a-z_]+\z/, message: "action names must be alphabetic snakecase" }
+
+    # Update items on our blacklist that have expired and can now be run
+    # def self.update_black_list
+    #   black_list = BlackListedAction.where.not(:duration_in_seconds => [nil,'',0])
+    #   black_list.each do |item|
+    #     target_time = item.created_at.to_i + item.duration_in_seconds
+    #     item.delete if target_time < Time.now.to_i
+    #   end 
+    # end
+
+    # Returns the time the blacklisting will be removed
+    # def finish_time
+    #   return nil if self.duration_in_seconds.nil?
+    #   Time.at(self.created_at.to_i + self.duration_in_seconds)
+    # end
+  end
+end

data/lib/cloud_crowd/models/node_record.rb

@@ -40,6 +40,10 @@ module CloudCrowd
         create!(attrs.merge(host_attr))
       end
     end
+    
+    def self.available_actions
+      available.map(&:actions).flatten.uniq - BlackListedAction.all.pluck(:action)
+    end
 
     # Dispatch a WorkUnit to this node. Places the node at back at the end of
     # the rotation. If we fail to send the WorkUnit, we consider the node to be

data/lib/cloud_crowd/models/work_unit.rb

@@ -42,7 +42,8 @@ module CloudCrowd
         # Find the available nodes, and determine what actions we're capable
         # of running at the moment.
         available_nodes   = NodeRecord.available.to_a
-        available_actions = available_nodes.map {|node| node.actions }.flatten.uniq
+        available_actions = NodeRecord.available_actions
+
         filter            = "action in (#{available_actions.map{|a| "'#{a}'"}.join(',')})"
         
         # If there aren't any available nodes or actions don't bother doing anything.

data/lib/cloud_crowd/schema.rb

@@ -26,6 +26,11 @@ ActiveRecord::Schema.define(:version => CloudCrowd::SCHEMA_VERSION) do
     t.datetime "updated_at"
   end
 
+  create_table "black_listed_actions", :force => true do |t|
+    t.string   "action",                              :null => false
+    t.datetime "created_at"
+  end
+
   create_table "work_units", :force => true do |t|
     t.integer  "status",                          :null => false
     t.integer  "job_id",                          :null => false

data/lib/cloud_crowd/server.rb

@@ -81,6 +81,31 @@ module CloudCrowd
       json current_job
     end
 
+    # Create a new blacklist item.
+    post '/blacklist' do
+      banned_action = BlackListedAction.new({:action=> params[:action]})
+      CloudCrowd.log("Blacklist ##{banned_action.id} (#{banned_action.action}) created.") unless ENV['RACK_ENV'] == 'test'
+      if banned_action.save
+        json banned_action
+      else
+        json banned_action.errors
+      end
+    end
+
+    # Delete a blacklist by action name
+    delete '/blacklist/:action' do
+      if action_ban = BlackListedAction.where(:action => params[:action]).first
+        json action_ban.destroy
+      else
+        raise not_found
+      end
+    end
+
+    # Retrieve a list of blacklisted actions and information about them.
+    get '/blacklist' do
+      json BlackListedAction.all
+    end
+
     # Cleans up a Job's saved S3 files. Delete a Job after you're done
     # downloading the results.
     delete '/jobs/:job_id' do

data/lib/cloud_crowd/version.rb

@@ -1,4 +1,4 @@
 module CloudCrowd
-  VERSION          = '0.7.5'
-  VERSION_RELEASED = '2015-06-17'
+  VERSION          = '0.7.6'
+  VERSION_RELEASED = '2015-10-28'
 end

data/public/css/admin_console.css

@@ -204,4 +204,22 @@ body {
       }
     .graph {
       height: 150px;
-    }
+    }
+    .info_table {
+      width: 300px;
+      outline: #555555 solid 2px;
+      text-align: center;
+      margin-left: 15px;
+    }
+      .info_table thead tr {
+        font-size: 16px;
+        color: #454545;
+        height: 50px;
+        background-color: white;
+      }
+      .info_table tbody tr {
+        height: 30px;
+        background-color: white;
+      }
+      
+

data/public/js/admin_console.js

@@ -44,6 +44,7 @@ window.Console = {
     this._workUnitCountEl   = $('#work_unit_count');
     this._nodeCountEl       = $('#node_count');
     this._workerCountEl     = $('#worker_count');
+    this._blacklistEl       = $('#blacklist');
     this._disconnected = $('#disconnected');
     $(window).bind('resize', Console.renderGraphs);
     $('#nodes .worker').live('click', Console.getWorkerInfo);
@@ -51,6 +52,7 @@ window.Console = {
     $('#workers_legend').css({background : this.WORKERS_COLOR});
     $('#nodes_legend').css({background : this.NODES_COLOR});
     this.getStatus();
+    this.getBlacklist();
     $.each(this.PRELOAD_IMAGES, function(){ var i = new Image(); i.src = this; });
   },
 
@@ -74,6 +76,27 @@ window.Console = {
     }});
   },
 
+  // Request the blacklisted actions from our server
+  getBlacklist : function () {
+    $.ajax({url : 'blacklist', dataType: 'json', success : function(resp) {
+      Console._blacklist      = resp;
+      if (Console._disconnected.is(':visible')) Console._disconnected.fadeOut(Console.ANIMATION_SPEED);
+      Console.renderBlacklist();
+      setTimeout(Console.getBlacklist, Console.POLL_INTERVAL);
+    }, error : function(request, status, errorThrown) {
+      if (!Console._disconnected.is(':visible')) Console._disconnected.fadeIn(Console.ANIMATION_SPEED);
+      setTimeout(Console.getBlacklist, Console.POLL_INTERVAL);
+    }});
+  },
+
+  // Render the blacklisted actions
+  renderBlacklist : function() {
+    Console._blacklistEl.empty()
+    this._blacklist.forEach(function(d){
+      Console._blacklistEl.append("<tr><td>" + d.action + "</td><td>" + d.created_at + "</td></tr>")
+    });
+  },
+
   // Fetch the last 100 lines of log from the server.
   tailLog : function() {
     $.ajax({url : 'log', success : function(resp) {

data/test/test_helper.rb

@@ -11,6 +11,7 @@ require 'shoulda/matchers/active_record'
 require 'shoulda/matchers/active_model'
 require 'machinist/active_record'
 require 'mocha/setup'
+require 'byebug'
 
 here = File.dirname(__FILE__)
 require File.expand_path(here + "/../lib/cloud-crowd")

data/test/unit/test_black_listed_action.rb

@@ -0,0 +1,33 @@
+require 'test_helper'
+
+class BlackListedActionTest < Minitest::Test
+  
+  context "A CloudCrowd::BlackListedAction" do
+        
+    setup do
+      BlackListedAction.destroy_all
+      @black_listed_action = BlackListedAction.create(action: 'word_count')
+      @node = NodeRecord.make!
+    end
+
+    teardown do
+      BlackListedAction.destroy_all
+    end
+    
+    should "fail if trying to create a duplicate blacklist entry" do
+      duplicate = BlackListedAction.create(action: 'word_count')
+      assert duplicate.errors.first[1] == "has already been taken"
+    end
+
+    should "create valid black listed action object" do
+      assert @black_listed_action.present?
+    end
+
+    should "not execute job because it is blacklisted" do
+      assert NodeRecord.all.map(&:actions).flatten.uniq.include? @black_listed_action.action
+      refute NodeRecord.available_actions.include? @black_listed_action.action
+    end
+
+  end
+  
+end

data/views/operations_center.erb

@@ -75,6 +75,22 @@
         </div>
         <div id="workers_graph" class="graph"></div>
       </div>
+      <div class="graph_container">
+        <div class="graph_title">
+          Blacklisted Actions
+        </div>
+        <table class="info_table">
+          <thead>
+            <tr class="heading">
+              <th>Name</th>
+              <th>Disabled Time</th>
+            </tr>
+          </thead>
+          <tbody id="blacklist">
+          </tbody>
+        </table>
+        <div id="actions_table" class="table"></div>
+      </div>
     </div>
 
     <div id="worker_info" style="display:none;">

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-crowd
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6
 platform: ruby
 authors:
 - Jeremy Ashkenas
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-17 00:00:00.000000000 Z
+date: 2015-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -221,6 +221,7 @@ files:
 - lib/cloud_crowd/helpers/resources.rb
 - lib/cloud_crowd/inflector.rb
 - lib/cloud_crowd/models.rb
+- lib/cloud_crowd/models/black_listed_action.rb
 - lib/cloud_crowd/models/job.rb
 - lib/cloud_crowd/models/node_record.rb
 - lib/cloud_crowd/models/work_unit.rb
@@ -261,6 +262,7 @@ files:
 - test/config/database.yml
 - test/test_helper.rb
 - test/unit/test_action.rb
+- test/unit/test_black_listed_action.rb
 - test/unit/test_command_line.rb
 - test/unit/test_configuration.rb
 - test/unit/test_job.rb
@@ -296,8 +298,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: cloud-crowd
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Parallel Processing for the Rest of Us
 test_files: []
+has_rdoc: