cloud-crowd 0.2.7 → 0.2.8

data/cloud-crowd.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.2.7'         # Keep version in sync with cloud-cloud.rb
-  s.date      = '2009-10-19'
+  s.version   = '0.2.8'         # Keep version in sync with cloud-cloud.rb
+  s.date      = '2009-10-27'
 
   s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
   s.summary     = "Parallel Processing for the Rest of Us"

data/lib/cloud-crowd.rb

@@ -44,7 +44,7 @@ module CloudCrowd
   autoload :WorkUnit,     'cloud_crowd/models'
   
   # Keep this version in sync with the gemspec.
-  VERSION        = '0.2.7'
+  VERSION        = '0.2.8'
   
   # Increment the schema version when there's a backwards incompatible change.
   SCHEMA_VERSION = 3
@@ -152,20 +152,25 @@ module CloudCrowd
     # Actions, then install only those into the actions directory.
     def actions
       return @actions if @actions
-      @actions = {}
-      default_actions   = Dir["#{ROOT}/actions/*.rb"]
-      installed_actions = Dir["#{@config_path}/actions/*.rb"]
-      custom_actions    = Dir["#{CloudCrowd.config[:actions_path]}/*.rb"]
-      (default_actions + installed_actions + custom_actions).each do |path|
+      @actions = action_paths.inject({}) do |memo, path|
         name = File.basename(path, File.extname(path))
         require path
-        @actions[name] = Module.const_get(Inflector.camelize(name))
+        memo[name] = Module.const_get(Inflector.camelize(name))
+        memo
       end
-      @actions
     rescue NameError => e
       adjusted_message = "One of your actions failed to load. Please ensure that the name of your action class can be deduced from the name of the file. ex: 'word_count.rb' => 'WordCount'\n#{e.message}" 
       raise NameError.new(adjusted_message, e.name)
     end
+    
+    # Retrieve the list of every installed Action for this node or server.
+    def action_paths
+      default_actions   = Dir["#{ROOT}/actions/*.rb"]
+      installed_actions = Dir["#{@config_path}/actions/*.rb"]
+      custom_actions    = CloudCrowd.config[:actions_path] ? Dir["#{CloudCrowd.config[:actions_path]}/*.rb"] : []
+      default_actions + installed_actions + custom_actions
+    end
+    
   end
   
 end

data/lib/cloud_crowd/models/job.rb

@@ -100,7 +100,7 @@ module CloudCrowd
     
     # This job is splittable if its Action has a +split+ method.
     def splittable?
-      self.action_class.public_instance_methods.include? 'split'
+      self.action_class.public_instance_methods.map {|m| m.to_sym }.include? :split
     end
     
     # This job is done splitting if it's finished with its splitting work units.
@@ -110,7 +110,7 @@ module CloudCrowd
     
     # This job is mergeable if its Action has a +merge+ method.
     def mergeable?
-      self.processing? && self.action_class.public_instance_methods.include?('merge')
+      self.processing? && self.action_class.public_instance_methods.map {|m| m.to_sym }.include?(:merge)
     end
     
     # Retrieve the class for this Job's Action.

data/lib/cloud_crowd/models/work_unit.rb

@@ -11,6 +11,10 @@ module CloudCrowd
     # The size of the maximum signed integer in MySQL -- SQLite has no limit.
     MAX_RESERVATION = 2147483647
     
+    # We only reserve a certain number of WorkUnits in a single go, to avoid
+    # reserving the entire table.
+    RESERVATION_LIMIT = 25
+    
     belongs_to :job
     belongs_to :node_record
     
@@ -28,32 +32,36 @@ module CloudCrowd
     # distributed to multiple nodes by reserving it first. The algorithm used
     # should be lock-free.
     #
-    # We loop over the WorkUnits reserved by this process and try to match them
-    # to Nodes that are capable of handling the Action. WorkUnits get removed 
-    # from the availability list when they are successfully sent, and Nodes get
-    # removed when they are busy or have the action in question disabled.
+    # We reserve WorkUnits for this process in chunks of RESERVATION_LIMIT size,
+    # and try to match them to Nodes that are capable of handling the Action. 
+    # WorkUnits get removed from the availability list when they are 
+    # successfully sent, and Nodes get removed when they are busy or have the 
+    # action in question disabled.
     def self.distribute_to_nodes
-      return unless reservation_number = WorkUnit.reserve_available
-      work_units = WorkUnit.reserved(reservation_number)
-      available_nodes = NodeRecord.available
-      while node = available_nodes.shift and unit = work_units.shift do
-        if node.actions.include? unit.action
-          if node.send_work_unit(unit)
-            available_nodes.push(node) unless node.busy?
-            next
+      begin
+        return unless reservation_number = WorkUnit.reserve_available(:limit => RESERVATION_LIMIT)
+        work_units = WorkUnit.reserved(reservation_number)
+        available_nodes = NodeRecord.available
+        while node = available_nodes.shift and unit = work_units.shift do
+          if node.actions.include? unit.action
+            if node.send_work_unit(unit)
+              available_nodes.push(node) unless node.busy?
+              next
+            end
           end
+          work_units.push(unit)
         end
-        work_units.push(unit)
+        retry if work_units.empty? && !available_nodes.empty?
+      ensure
+        WorkUnit.cancel_reservations(reservation_number) if reservation_number
       end
-    ensure
-      WorkUnit.cancel_reservations(reservation_number) if reservation_number
     end
     
     # Reserves all available WorkUnits for this process. Returns false if there 
     # were none available.
-    def self.reserve_available
+    def self.reserve_available(options={})
       reservation_number = ActiveSupport::SecureRandom.random_number(MAX_RESERVATION)
-      any = WorkUnit.available.update_all("reservation = #{reservation_number}") > 0
+      any = WorkUnit.available.update_all("reservation = #{reservation_number}", nil, options) > 0
       any && reservation_number
     end
     

data/test/unit/test_configuration.rb

@@ -3,7 +3,9 @@ require 'test_helper'
 class ConfigurationTest < Test::Unit::TestCase
 
   context "CloudCrowd Configuration" do
-            
+    
+    setup { CloudCrowd.instance_variable_set("@actions", nil) }
+
     should "have read in config.yml" do
       assert CloudCrowd.config[:max_workers] == 10
       assert CloudCrowd.config[:storage] == 'filesystem'
@@ -22,6 +24,17 @@ class ConfigurationTest < Test::Unit::TestCase
       assert CloudCrowd.actions['process_pdfs']     == ProcessPdfs
       assert CloudCrowd.actions['graphics_magick']  == GraphicsMagick
     end
+    
+    should "not find custom actions unless 'actions_path' is set" do
+      CloudCrowd.config[:actions_path] = nil
+      assert CloudCrowd.actions.keys.length == 4
+    end
+    
+    should "find custom actions when 'actions_path' is set" do
+      CloudCrowd.config[:actions_path] = "#{CloudCrowd::ROOT}/test/config/actions/custom"
+      assert CloudCrowd.actions['echo_action'] == EchoAction
+      assert CloudCrowd.actions.keys.length == 5
+    end
             
   end
   

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: cloud-crowd
 version: !ruby/object:Gem::Version 
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors: 
 - Jeremy Ashkenas
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-19 00:00:00 -04:00
+date: 2009-10-27 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency