cloud-crowd 0.2.8 → 0.2.9

data/actions/process_pdfs.rb

@@ -21,7 +21,7 @@ class ProcessPdfs < CloudCrowd::Action
       batch_pdfs = pdfs[batch_num*batch_size...(batch_num + 1)*batch_size]
       `tar -czf #{tar_path} #{batch_pdfs.join(' ')}`
     end
-    Dir["*.tar"].map {|tar| save(tar) }.to_json
+    Dir["*.tar"].map {|tar| save(tar) }
   end
 
   # Convert a pdf page into different-sized thumbnails. Grab the text.

data/cloud-crowd.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.2.8'         # Keep version in sync with cloud-cloud.rb
-  s.date      = '2009-10-27'
+  s.version   = '0.2.9'         # Keep version in sync with cloud-cloud.rb
+  s.date      = '2009-11-03'
 
   s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
   s.summary     = "Parallel Processing for the Rest of Us"

data/lib/cloud-crowd.rb

@@ -44,7 +44,7 @@ module CloudCrowd
   autoload :WorkUnit,     'cloud_crowd/models'
   
   # Keep this version in sync with the gemspec.
-  VERSION        = '0.2.8'
+  VERSION        = '0.2.9'
   
   # Increment the schema version when there's a backwards incompatible change.
   SCHEMA_VERSION = 3
@@ -87,6 +87,7 @@ module CloudCrowd
   
   class << self
     attr_reader :config
+    attr_accessor :identity
     
     # Configure CloudCrowd by passing in the path to <tt>config.yml</tt>.
     def configure(config_path)
@@ -171,6 +172,17 @@ module CloudCrowd
       default_actions + installed_actions + custom_actions
     end
     
+    # Is this CloudCrowd instance a server? Useful for avoiding loading unneeded
+    # code from actions.
+    def server?
+      @identity == :server
+    end
+    
+    # Or is it a node?
+    def node?
+      @identity == :node
+    end
+    
   end
   
 end

data/lib/cloud_crowd/action.rb

@@ -31,7 +31,8 @@ module CloudCrowd
       @job_id, @work_unit_id = options['job_id'], options['work_unit_id']
       @work_directory = File.expand_path(File.join(@store.temp_storage_path, storage_prefix))
       FileUtils.mkdir_p(@work_directory) unless File.exists?(@work_directory)
-      status == MERGING ? parse_input : download_input
+      parse_input
+      download_input
     end
     
     # Each Action subclass must implement a +process+ method, overriding this.
@@ -98,9 +99,11 @@ module CloudCrowd
       @storage_prefix ||= File.join(path_parts)
     end
     
-    # If we know that the input is JSON, replace it with the parsed form.
+    # If we think that the input is JSON, replace it with the parsed form.
+    # It would be great if the JSON module had an is_json? method.
     def parse_input
-      @input = JSON.parse(@input)
+      return unless ['[', '{'].include? @input[0..0]
+      @input = JSON.parse(@input) rescue @input
     end
     
     def input_is_url?

data/lib/cloud_crowd/models/work_unit.rb

@@ -23,8 +23,8 @@ module CloudCrowd
     # Available WorkUnits are waiting to be distributed to Nodes for processing.
     named_scope :available, {:conditions => {:reservation => nil, :worker_pid => nil, :status => INCOMPLETE}}
     # Reserved WorkUnits have been marked for distribution by a central server process.
-    named_scope :reserved,  lambda {|reservation_number| 
-      {:conditions => {:reservation => reservation_number}, :order => 'updated_at asc'}
+    named_scope :reserved,  lambda {|reservation| 
+      {:conditions => {:reservation => reservation}, :order => 'updated_at asc'}
     }
     
     # Attempt to send a list of WorkUnits to nodes with available capacity.
@@ -38,9 +38,10 @@ module CloudCrowd
     # successfully sent, and Nodes get removed when they are busy or have the 
     # action in question disabled.
     def self.distribute_to_nodes
-      begin
-        return unless reservation_number = WorkUnit.reserve_available(:limit => RESERVATION_LIMIT)
-        work_units = WorkUnit.reserved(reservation_number)
+      reservation = nil
+      loop do
+        return unless reservation = WorkUnit.reserve_available(:limit => RESERVATION_LIMIT)
+        work_units = WorkUnit.reserved(reservation)
         available_nodes = NodeRecord.available
         while node = available_nodes.shift and unit = work_units.shift do
           if node.actions.include? unit.action
@@ -51,23 +52,23 @@ module CloudCrowd
           end
           work_units.push(unit)
         end
-        retry if work_units.empty? && !available_nodes.empty?
-      ensure
-        WorkUnit.cancel_reservations(reservation_number) if reservation_number
+        return if work_units.any? || available_nodes.empty?
       end
+    ensure
+      WorkUnit.cancel_reservations(reservation) if reservation
     end
     
     # Reserves all available WorkUnits for this process. Returns false if there 
     # were none available.
     def self.reserve_available(options={})
-      reservation_number = ActiveSupport::SecureRandom.random_number(MAX_RESERVATION)
-      any = WorkUnit.available.update_all("reservation = #{reservation_number}", nil, options) > 0
-      any && reservation_number
+      reservation = ActiveSupport::SecureRandom.random_number(MAX_RESERVATION)
+      any = WorkUnit.available.update_all("reservation = #{reservation}", nil, options) > 0
+      any && reservation
     end
     
     # Cancels all outstanding WorkUnit reservations for this process.
-    def self.cancel_reservations(reservation_number)
-      WorkUnit.reserved(reservation_number).update_all('reservation = null')
+    def self.cancel_reservations(reservation)
+      WorkUnit.reserved(reservation).update_all('reservation = null')
     end
     
     # Cancels all outstanding WorkUnit reservations for all processes. (Useful
@@ -95,7 +96,8 @@ module CloudCrowd
     # without waiting for the rest of their splitting cousins to complete.
     def finish(result, time_taken)
       if splitting?
-        [JSON.parse(parsed_output(result))].flatten.each do |new_input|
+        [parsed_output(result)].flatten.each do |new_input|
+          new_input = new_input.to_json unless new_input.is_a? String
           WorkUnit.start(job, action, new_input, PROCESSING)
         end
         self.destroy

data/lib/cloud_crowd/node.rb

@@ -65,6 +65,7 @@ module CloudCrowd
     # When creating a node, specify the port it should run on.
     def initialize(port=nil, daemon=false)
       require 'json'
+      CloudCrowd.identity = :node
       @central          = CloudCrowd.central_server
       @host             = Socket.gethostname
       @enabled_actions  = CloudCrowd.actions.keys

data/lib/cloud_crowd/server.rb

@@ -112,6 +112,12 @@ module CloudCrowd
       json nil
     end
     
+    # At initialization record the identity of this Ruby instance as a server.
+    def initialize(*args)
+      super(*args)
+      CloudCrowd.identity = :server
+    end
+    
   end
   
 end

data/test/acceptance/test_server.rb

@@ -15,6 +15,11 @@ class ServerTest < Test::Unit::TestCase
       2.times { Job.make }
     end
     
+    should "set the identity of the Ruby instance" do
+      app.new
+      assert CloudCrowd.server?
+    end
+    
     should "be able to render the Operations Center (GET /)" do
       get '/'
       assert last_response.body.include? '<div id="nodes">'
@@ -29,18 +34,6 @@ class ServerTest < Test::Unit::TestCase
       assert resp['work_unit_count'] == 2
     end
     
-    # should "be able to check in a worker daemon, and then check out a work unit" do
-    #   put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
-    #   assert last_response.successful? && last_response.empty?
-    #   post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
-    #   checked_out = JSON.parse(last_response.body)
-    #   assert checked_out['action'] == 'graphics_magick'
-    #   assert checked_out['attempts'] == 0
-    #   assert checked_out['status'] == CloudCrowd::PROCESSING
-    #   status_check = JSON.parse(get('/worker/101@localhost').body)
-    #   assert checked_out == status_check
-    # end
-    
     should "have a heartbeat" do
       assert get('/heartbeat').body == 'buh-bump'
     end

data/test/unit/test_node.rb

@@ -8,6 +8,10 @@ class NodeUnitTest < Test::Unit::TestCase
       @node = Node.new(11011).instance_variable_get(:@app)
     end
     
+    should "set the identity of the Ruby instance" do
+      assert CloudCrowd.node?
+    end
+    
     should "instantiate correctly" do
       assert @node.central.to_s == "http://localhost:9173"
       assert @node.port == 11011

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: cloud-crowd
 version: !ruby/object:Gem::Version 
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors: 
 - Jeremy Ashkenas
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-27 00:00:00 -04:00
+date: 2009-11-03 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency