cloud-crowd 0.7.1 → 0.7.2.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f881c2662b37b888ae7f2879e7fa1a55cc6f5a2
-  data.tar.gz: 943946e48251445f413bb14a62f6eddcda212008
+  metadata.gz: 80888bac2f20a7754fce2189300bc2eee29d9d83
+  data.tar.gz: b972fd367a79015365f3a8d96390f2a968703faf
 SHA512:
-  metadata.gz: 2ae8ab76fdc5cc9157a82717c3edd2b48c7ae5ec1e23f04fbc4babebc5421232ff6d417acd7dc6be668f682edad577ae6df8cece452d5307ee0382d20a8a7527
-  data.tar.gz: 2df7d0d9e2b9139c71f67ba2f782d7f31c2ab61908d9b6434f3e144a6745fc20159f8b1ad4217a0e045b3fe7dc92bb88fce5b516540ea9211016c9921a1f54a3
+  metadata.gz: 8bc1b167b09984f551a20110ab40294f5a7da6b6843abb0f49c5b753309e5c85e793f8f1d93d7f97e27832b726e9a7989a27bb64a64dbe0aa8442ef2d0dee211
+  data.tar.gz: 434cc845d94bc70d01c0d5aee4af0a82b8e3ff5b54442432b307dd5a83bb3ef3ad0d11cad0b6ebf75bcac35d61b57982d2fce0ea8a5b6f49fe29d04e977741df

data/cloud-crowd.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name      = 'cloud-crowd'
-  s.version   = '0.7.1'         # Keep version in sync with cloud-cloud.rb
-  s.date      = '2014-04-05'
+  s.version   = '0.7.2.beta'         # Keep version in sync with cloud-cloud.rb
+  s.date      = '2014-04-06'
 
   s.homepage    = "http://wiki.github.com/documentcloud/cloud-crowd"
   s.summary     = "Parallel Processing for the Rest of Us"
@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'json',          ['>= 1.1.7']
   s.add_dependency 'rest-client',   ['>= 1.4']
   s.add_dependency 'thin',          ['>= 1.2.4']
-  s.add_dependency 'rake'  
+  s.add_dependency 'rake'
 
   if s.respond_to?(:add_development_dependency)
     s.add_development_dependency 'faker',              ['>= 0.3.1']

data/lib/cloud-crowd.rb

@@ -47,7 +47,7 @@ module CloudCrowd
   autoload :WorkUnit,     'cloud_crowd/models'
 
   # Keep this version in sync with the gemspec.
-  VERSION        = '0.7.1'
+  VERSION        = '0.7.2'
 
   # Increment the schema version when there's a backwards incompatible change.
   SCHEMA_VERSION = 4
@@ -100,6 +100,10 @@ module CloudCrowd
       @config_path = File.expand_path(File.dirname(config_path))
       @config = YAML.load(ERB.new(File.read(config_path)).result)
       @config[:work_unit_retries] ||= MIN_RETRIES
+      if @config[:actions_path]
+        path = Pathname.new( @config[:actions_path] ).realpath
+        $LOAD_PATH.unshift( path ) unless $LOAD_PATH.include?( path )
+      end
     end
 
     # Configure the CloudCrowd central database (and connect to it), by passing
@@ -116,6 +120,16 @@ module CloudCrowd
       end
     end
 
+    # Starts a new thread with a ActiveRecord connection_pool
+    # and yields for peforming work inside the blocks
+    def defer
+      Thread.new do
+        ActiveRecord::Base.connection_pool.with_connection do
+          yield
+        end
+      end
+    end
+
     # Get a reference to the central server, including authentication if
     # configured.
     def central_server
@@ -164,9 +178,10 @@ module CloudCrowd
     def actions
       return @actions if @actions
       @actions = action_paths.inject({}) do |memo, path|
-        name = File.basename(path, File.extname(path))
-        require path
-        memo[name] = Module.const_get(Inflector.camelize(name))
+        path = Pathname.new(path)
+        require path.relative? ? path.basename : path
+        name = path.basename('.*').to_s
+        memo[name] = Module.const_get( Inflector.camelize( name ) )
         memo
       end
     rescue NameError => e

data/lib/cloud_crowd/asset_store/s3_store.rb

@@ -1,4 +1,4 @@
-gem 'right_aws'
+require 'aws'
 
 module CloudCrowd
   class AssetStore
@@ -14,11 +14,8 @@ module CloudCrowd
         key, secret = CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key]
         valid_conf  = [bucket_name, key, secret].all? {|s| s.is_a? String }
         raise Error::MissingConfiguration, "An S3 account must be configured in 'config.yml' before 's3' storage can be used" unless valid_conf
-        protocol    = @use_auth ? 'https' : 'http'
-        port        = @use_auth ? 443 : 80
-        @s3         = RightAws::S3.new(key, secret, :protocol => protocol, :port => port)
-        @bucket     = @s3.bucket(bucket_name)
-        @bucket     = @s3.bucket(bucket_name, true) unless @bucket
+        @s3         = ::AWS::S3.new(:access_key_id => key, :secret_access_key => secret, :secure => !!@use_auth)
+        @bucket     = (@s3.buckets[bucket_name].exists? ? @s3.buckets[bucket_name] : @s3.buckets.create(bucket_name))
       end
 
       # Save a finished file from local storage to S3. Save it publicly unless

data/lib/cloud_crowd/models/job.rb

@@ -54,7 +54,7 @@ module CloudCrowd
       if complete?
         update_attributes(:outputs => outs, :time => time_taken)
         puts "Job ##{id} (#{action}) #{display_status}." unless ENV['RACK_ENV'] == 'test'
-        Thread.new { fire_callback } if callback_url
+        CloudCrowd.defer { fire_callback } if callback_url
       end
       self
     end
@@ -80,7 +80,7 @@ module CloudCrowd
     def fire_callback
       begin
         response = RestClient.post(callback_url, {:job => self.to_json})
-        Thread.new { self.destroy } if response && response.code == 201
+        CloudCrowd.defer { self.destroy } if response && response.code == 201
       rescue RestClient::Exception => e
         puts "Job ##{id} (#{action}) failed to fire callback: #{callback_url}"
       end
@@ -188,4 +188,4 @@ module CloudCrowd
     end
 
   end
-end
+end

data/lib/cloud_crowd/models/node_record.rb

@@ -116,7 +116,7 @@ module CloudCrowd
     # Redistribute in a separate thread to avoid delaying shutdown.
     def redistribute_work_units
       release_work_units
-      Thread.new { WorkUnit.distribute_to_nodes }
+      CloudCrowd.defer { WorkUnit.distribute_to_nodes }
     end
 
   end

data/lib/cloud_crowd/node.rb

@@ -65,6 +65,7 @@ module CloudCrowd
 
     # When creating a node, specify the port it should run on.
     def initialize(options={})
+      super()
       require 'json'
       CloudCrowd.identity = :node
       @central          = CloudCrowd.central_server
@@ -91,7 +92,7 @@ module CloudCrowd
       @server.daemonize if @daemon
       trap_signals
       asset_store
-      @server_thread   = Thread.new { @server.start }
+      @server_thread   = CloudCrowd.defer { @server.start }
       check_in(true)
       check_in_periodically
       monitor_system if @max_load || @min_memory
@@ -158,7 +159,7 @@ module CloudCrowd
     # average and the amount of free memory remaining. If we transition out of
     # the overloaded state, let central know.
     def monitor_system
-      @monitor_thread = Thread.new do
+      @monitor_thread = CloudCrowd.defer do
         loop do
           was_overloaded = @overloaded
           @overloaded = overloaded?
@@ -172,7 +173,7 @@ module CloudCrowd
     # will assume that the node has gone down. Checking in will let central know
     # it's still online.
     def check_in_periodically
-      @check_in_thread = Thread.new do
+      @check_in_thread = CloudCrowd.defer do
         loop do
           sleep CHECK_IN_INTERVAL
           check_in

data/lib/cloud_crowd/server.rb

@@ -71,7 +71,7 @@ module CloudCrowd
     # Distributes all work units to available nodes.
     post '/jobs' do
       job = Job.create_from_request(JSON.parse(params[:job]))
-      Thread.new { WorkUnit.distribute_to_nodes }
+      CloudCrowd.defer { WorkUnit.distribute_to_nodes }
       puts "Job ##{job.id} (#{job.action}) started." unless ENV['RACK_ENV'] == 'test'
       json job
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud-crowd
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2.beta
 platform: ruby
 authors:
 - Jeremy Ashkenas
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-05 00:00:00.000000000 Z
+date: 2014-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -288,14 +288,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: cloud-crowd
-rubygems_version: 2.2.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Parallel Processing for the Rest of Us
 test_files: []
-has_rdoc: