cloud-crowd 0.7.1 → 0.7.2.beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9f881c2662b37b888ae7f2879e7fa1a55cc6f5a2
4
- data.tar.gz: 943946e48251445f413bb14a62f6eddcda212008
3
+ metadata.gz: 80888bac2f20a7754fce2189300bc2eee29d9d83
4
+ data.tar.gz: b972fd367a79015365f3a8d96390f2a968703faf
5
5
  SHA512:
6
- metadata.gz: 2ae8ab76fdc5cc9157a82717c3edd2b48c7ae5ec1e23f04fbc4babebc5421232ff6d417acd7dc6be668f682edad577ae6df8cece452d5307ee0382d20a8a7527
7
- data.tar.gz: 2df7d0d9e2b9139c71f67ba2f782d7f31c2ab61908d9b6434f3e144a6745fc20159f8b1ad4217a0e045b3fe7dc92bb88fce5b516540ea9211016c9921a1f54a3
6
+ metadata.gz: 8bc1b167b09984f551a20110ab40294f5a7da6b6843abb0f49c5b753309e5c85e793f8f1d93d7f97e27832b726e9a7989a27bb64a64dbe0aa8442ef2d0dee211
7
+ data.tar.gz: 434cc845d94bc70d01c0d5aee4af0a82b8e3ff5b54442432b307dd5a83bb3ef3ad0d11cad0b6ebf75bcac35d61b57982d2fce0ea8a5b6f49fe29d04e977741df
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'cloud-crowd'
3
- s.version = '0.7.1' # Keep version in sync with cloud-cloud.rb
4
- s.date = '2014-04-05'
3
+ s.version = '0.7.2.beta' # Keep version in sync with cloud-cloud.rb
4
+ s.date = '2014-04-06'
5
5
 
6
6
  s.homepage = "http://wiki.github.com/documentcloud/cloud-crowd"
7
7
  s.summary = "Parallel Processing for the Rest of Us"
@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
34
34
  s.add_dependency 'json', ['>= 1.1.7']
35
35
  s.add_dependency 'rest-client', ['>= 1.4']
36
36
  s.add_dependency 'thin', ['>= 1.2.4']
37
- s.add_dependency 'rake'
37
+ s.add_dependency 'rake'
38
38
 
39
39
  if s.respond_to?(:add_development_dependency)
40
40
  s.add_development_dependency 'faker', ['>= 0.3.1']
@@ -47,7 +47,7 @@ module CloudCrowd
47
47
  autoload :WorkUnit, 'cloud_crowd/models'
48
48
 
49
49
  # Keep this version in sync with the gemspec.
50
- VERSION = '0.7.1'
50
+ VERSION = '0.7.2'
51
51
 
52
52
  # Increment the schema version when there's a backwards incompatible change.
53
53
  SCHEMA_VERSION = 4
@@ -100,6 +100,10 @@ module CloudCrowd
100
100
  @config_path = File.expand_path(File.dirname(config_path))
101
101
  @config = YAML.load(ERB.new(File.read(config_path)).result)
102
102
  @config[:work_unit_retries] ||= MIN_RETRIES
103
+ if @config[:actions_path]
104
+ path = Pathname.new( @config[:actions_path] ).realpath
105
+ $LOAD_PATH.unshift( path ) unless $LOAD_PATH.include?( path )
106
+ end
103
107
  end
104
108
 
105
109
  # Configure the CloudCrowd central database (and connect to it), by passing
@@ -116,6 +120,16 @@ module CloudCrowd
116
120
  end
117
121
  end
118
122
 
123
+ # Starts a new thread with a ActiveRecord connection_pool
124
+ # and yields for peforming work inside the blocks
125
+ def defer
126
+ Thread.new do
127
+ ActiveRecord::Base.connection_pool.with_connection do
128
+ yield
129
+ end
130
+ end
131
+ end
132
+
119
133
  # Get a reference to the central server, including authentication if
120
134
  # configured.
121
135
  def central_server
@@ -164,9 +178,10 @@ module CloudCrowd
164
178
  def actions
165
179
  return @actions if @actions
166
180
  @actions = action_paths.inject({}) do |memo, path|
167
- name = File.basename(path, File.extname(path))
168
- require path
169
- memo[name] = Module.const_get(Inflector.camelize(name))
181
+ path = Pathname.new(path)
182
+ require path.relative? ? path.basename : path
183
+ name = path.basename('.*').to_s
184
+ memo[name] = Module.const_get( Inflector.camelize( name ) )
170
185
  memo
171
186
  end
172
187
  rescue NameError => e
@@ -1,4 +1,4 @@
1
- gem 'right_aws'
1
+ require 'aws'
2
2
 
3
3
  module CloudCrowd
4
4
  class AssetStore
@@ -14,11 +14,8 @@ module CloudCrowd
14
14
  key, secret = CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key]
15
15
  valid_conf = [bucket_name, key, secret].all? {|s| s.is_a? String }
16
16
  raise Error::MissingConfiguration, "An S3 account must be configured in 'config.yml' before 's3' storage can be used" unless valid_conf
17
- protocol = @use_auth ? 'https' : 'http'
18
- port = @use_auth ? 443 : 80
19
- @s3 = RightAws::S3.new(key, secret, :protocol => protocol, :port => port)
20
- @bucket = @s3.bucket(bucket_name)
21
- @bucket = @s3.bucket(bucket_name, true) unless @bucket
17
+ @s3 = ::AWS::S3.new(:access_key_id => key, :secret_access_key => secret, :secure => !!@use_auth)
18
+ @bucket = (@s3.buckets[bucket_name].exists? ? @s3.buckets[bucket_name] : @s3.buckets.create(bucket_name))
22
19
  end
23
20
 
24
21
  # Save a finished file from local storage to S3. Save it publicly unless
@@ -54,7 +54,7 @@ module CloudCrowd
54
54
  if complete?
55
55
  update_attributes(:outputs => outs, :time => time_taken)
56
56
  puts "Job ##{id} (#{action}) #{display_status}." unless ENV['RACK_ENV'] == 'test'
57
- Thread.new { fire_callback } if callback_url
57
+ CloudCrowd.defer { fire_callback } if callback_url
58
58
  end
59
59
  self
60
60
  end
@@ -80,7 +80,7 @@ module CloudCrowd
80
80
  def fire_callback
81
81
  begin
82
82
  response = RestClient.post(callback_url, {:job => self.to_json})
83
- Thread.new { self.destroy } if response && response.code == 201
83
+ CloudCrowd.defer { self.destroy } if response && response.code == 201
84
84
  rescue RestClient::Exception => e
85
85
  puts "Job ##{id} (#{action}) failed to fire callback: #{callback_url}"
86
86
  end
@@ -188,4 +188,4 @@ module CloudCrowd
188
188
  end
189
189
 
190
190
  end
191
- end
191
+ end
@@ -116,7 +116,7 @@ module CloudCrowd
116
116
  # Redistribute in a separate thread to avoid delaying shutdown.
117
117
  def redistribute_work_units
118
118
  release_work_units
119
- Thread.new { WorkUnit.distribute_to_nodes }
119
+ CloudCrowd.defer { WorkUnit.distribute_to_nodes }
120
120
  end
121
121
 
122
122
  end
@@ -65,6 +65,7 @@ module CloudCrowd
65
65
 
66
66
  # When creating a node, specify the port it should run on.
67
67
  def initialize(options={})
68
+ super()
68
69
  require 'json'
69
70
  CloudCrowd.identity = :node
70
71
  @central = CloudCrowd.central_server
@@ -91,7 +92,7 @@ module CloudCrowd
91
92
  @server.daemonize if @daemon
92
93
  trap_signals
93
94
  asset_store
94
- @server_thread = Thread.new { @server.start }
95
+ @server_thread = CloudCrowd.defer { @server.start }
95
96
  check_in(true)
96
97
  check_in_periodically
97
98
  monitor_system if @max_load || @min_memory
@@ -158,7 +159,7 @@ module CloudCrowd
158
159
  # average and the amount of free memory remaining. If we transition out of
159
160
  # the overloaded state, let central know.
160
161
  def monitor_system
161
- @monitor_thread = Thread.new do
162
+ @monitor_thread = CloudCrowd.defer do
162
163
  loop do
163
164
  was_overloaded = @overloaded
164
165
  @overloaded = overloaded?
@@ -172,7 +173,7 @@ module CloudCrowd
172
173
  # will assume that the node has gone down. Checking in will let central know
173
174
  # it's still online.
174
175
  def check_in_periodically
175
- @check_in_thread = Thread.new do
176
+ @check_in_thread = CloudCrowd.defer do
176
177
  loop do
177
178
  sleep CHECK_IN_INTERVAL
178
179
  check_in
@@ -71,7 +71,7 @@ module CloudCrowd
71
71
  # Distributes all work units to available nodes.
72
72
  post '/jobs' do
73
73
  job = Job.create_from_request(JSON.parse(params[:job]))
74
- Thread.new { WorkUnit.distribute_to_nodes }
74
+ CloudCrowd.defer { WorkUnit.distribute_to_nodes }
75
75
  puts "Job ##{job.id} (#{job.action}) started." unless ENV['RACK_ENV'] == 'test'
76
76
  json job
77
77
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cloud-crowd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1
4
+ version: 0.7.2.beta
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Ashkenas
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2014-04-05 00:00:00.000000000 Z
12
+ date: 2014-04-06 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activerecord
@@ -288,14 +288,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
288
288
  version: '0'
289
289
  required_rubygems_version: !ruby/object:Gem::Requirement
290
290
  requirements:
291
- - - ">="
291
+ - - ">"
292
292
  - !ruby/object:Gem::Version
293
- version: '0'
293
+ version: 1.3.1
294
294
  requirements: []
295
295
  rubyforge_project: cloud-crowd
296
- rubygems_version: 2.2.1
296
+ rubygems_version: 2.2.2
297
297
  signing_key:
298
298
  specification_version: 4
299
299
  summary: Parallel Processing for the Rest of Us
300
300
  test_files: []
301
- has_rdoc: