cloud-crowd 0.7.1 → 0.7.2.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/cloud-crowd.gemspec +3 -3
- data/lib/cloud-crowd.rb +19 -4
- data/lib/cloud_crowd/asset_store/s3_store.rb +3 -6
- data/lib/cloud_crowd/models/job.rb +3 -3
- data/lib/cloud_crowd/models/node_record.rb +1 -1
- data/lib/cloud_crowd/node.rb +4 -3
- data/lib/cloud_crowd/server.rb +1 -1
- metadata +5 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 80888bac2f20a7754fce2189300bc2eee29d9d83
|
4
|
+
data.tar.gz: b972fd367a79015365f3a8d96390f2a968703faf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bc1b167b09984f551a20110ab40294f5a7da6b6843abb0f49c5b753309e5c85e793f8f1d93d7f97e27832b726e9a7989a27bb64a64dbe0aa8442ef2d0dee211
|
7
|
+
data.tar.gz: 434cc845d94bc70d01c0d5aee4af0a82b8e3ff5b54442432b307dd5a83bb3ef3ad0d11cad0b6ebf75bcac35d61b57982d2fce0ea8a5b6f49fe29d04e977741df
|
data/cloud-crowd.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'cloud-crowd'
|
3
|
-
s.version = '0.7.
|
4
|
-
s.date = '2014-04-
|
3
|
+
s.version = '0.7.2.beta' # Keep version in sync with cloud-cloud.rb
|
4
|
+
s.date = '2014-04-06'
|
5
5
|
|
6
6
|
s.homepage = "http://wiki.github.com/documentcloud/cloud-crowd"
|
7
7
|
s.summary = "Parallel Processing for the Rest of Us"
|
@@ -34,7 +34,7 @@ Gem::Specification.new do |s|
|
|
34
34
|
s.add_dependency 'json', ['>= 1.1.7']
|
35
35
|
s.add_dependency 'rest-client', ['>= 1.4']
|
36
36
|
s.add_dependency 'thin', ['>= 1.2.4']
|
37
|
-
s.add_dependency 'rake'
|
37
|
+
s.add_dependency 'rake'
|
38
38
|
|
39
39
|
if s.respond_to?(:add_development_dependency)
|
40
40
|
s.add_development_dependency 'faker', ['>= 0.3.1']
|
data/lib/cloud-crowd.rb
CHANGED
@@ -47,7 +47,7 @@ module CloudCrowd
|
|
47
47
|
autoload :WorkUnit, 'cloud_crowd/models'
|
48
48
|
|
49
49
|
# Keep this version in sync with the gemspec.
|
50
|
-
VERSION = '0.7.
|
50
|
+
VERSION = '0.7.2'
|
51
51
|
|
52
52
|
# Increment the schema version when there's a backwards incompatible change.
|
53
53
|
SCHEMA_VERSION = 4
|
@@ -100,6 +100,10 @@ module CloudCrowd
|
|
100
100
|
@config_path = File.expand_path(File.dirname(config_path))
|
101
101
|
@config = YAML.load(ERB.new(File.read(config_path)).result)
|
102
102
|
@config[:work_unit_retries] ||= MIN_RETRIES
|
103
|
+
if @config[:actions_path]
|
104
|
+
path = Pathname.new( @config[:actions_path] ).realpath
|
105
|
+
$LOAD_PATH.unshift( path ) unless $LOAD_PATH.include?( path )
|
106
|
+
end
|
103
107
|
end
|
104
108
|
|
105
109
|
# Configure the CloudCrowd central database (and connect to it), by passing
|
@@ -116,6 +120,16 @@ module CloudCrowd
|
|
116
120
|
end
|
117
121
|
end
|
118
122
|
|
123
|
+
# Starts a new thread with a ActiveRecord connection_pool
|
124
|
+
# and yields for peforming work inside the blocks
|
125
|
+
def defer
|
126
|
+
Thread.new do
|
127
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
128
|
+
yield
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
119
133
|
# Get a reference to the central server, including authentication if
|
120
134
|
# configured.
|
121
135
|
def central_server
|
@@ -164,9 +178,10 @@ module CloudCrowd
|
|
164
178
|
def actions
|
165
179
|
return @actions if @actions
|
166
180
|
@actions = action_paths.inject({}) do |memo, path|
|
167
|
-
|
168
|
-
require path
|
169
|
-
|
181
|
+
path = Pathname.new(path)
|
182
|
+
require path.relative? ? path.basename : path
|
183
|
+
name = path.basename('.*').to_s
|
184
|
+
memo[name] = Module.const_get( Inflector.camelize( name ) )
|
170
185
|
memo
|
171
186
|
end
|
172
187
|
rescue NameError => e
|
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
require 'aws'
|
2
2
|
|
3
3
|
module CloudCrowd
|
4
4
|
class AssetStore
|
@@ -14,11 +14,8 @@ module CloudCrowd
|
|
14
14
|
key, secret = CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key]
|
15
15
|
valid_conf = [bucket_name, key, secret].all? {|s| s.is_a? String }
|
16
16
|
raise Error::MissingConfiguration, "An S3 account must be configured in 'config.yml' before 's3' storage can be used" unless valid_conf
|
17
|
-
|
18
|
-
|
19
|
-
@s3 = RightAws::S3.new(key, secret, :protocol => protocol, :port => port)
|
20
|
-
@bucket = @s3.bucket(bucket_name)
|
21
|
-
@bucket = @s3.bucket(bucket_name, true) unless @bucket
|
17
|
+
@s3 = ::AWS::S3.new(:access_key_id => key, :secret_access_key => secret, :secure => !!@use_auth)
|
18
|
+
@bucket = (@s3.buckets[bucket_name].exists? ? @s3.buckets[bucket_name] : @s3.buckets.create(bucket_name))
|
22
19
|
end
|
23
20
|
|
24
21
|
# Save a finished file from local storage to S3. Save it publicly unless
|
@@ -54,7 +54,7 @@ module CloudCrowd
|
|
54
54
|
if complete?
|
55
55
|
update_attributes(:outputs => outs, :time => time_taken)
|
56
56
|
puts "Job ##{id} (#{action}) #{display_status}." unless ENV['RACK_ENV'] == 'test'
|
57
|
-
|
57
|
+
CloudCrowd.defer { fire_callback } if callback_url
|
58
58
|
end
|
59
59
|
self
|
60
60
|
end
|
@@ -80,7 +80,7 @@ module CloudCrowd
|
|
80
80
|
def fire_callback
|
81
81
|
begin
|
82
82
|
response = RestClient.post(callback_url, {:job => self.to_json})
|
83
|
-
|
83
|
+
CloudCrowd.defer { self.destroy } if response && response.code == 201
|
84
84
|
rescue RestClient::Exception => e
|
85
85
|
puts "Job ##{id} (#{action}) failed to fire callback: #{callback_url}"
|
86
86
|
end
|
@@ -188,4 +188,4 @@ module CloudCrowd
|
|
188
188
|
end
|
189
189
|
|
190
190
|
end
|
191
|
-
end
|
191
|
+
end
|
@@ -116,7 +116,7 @@ module CloudCrowd
|
|
116
116
|
# Redistribute in a separate thread to avoid delaying shutdown.
|
117
117
|
def redistribute_work_units
|
118
118
|
release_work_units
|
119
|
-
|
119
|
+
CloudCrowd.defer { WorkUnit.distribute_to_nodes }
|
120
120
|
end
|
121
121
|
|
122
122
|
end
|
data/lib/cloud_crowd/node.rb
CHANGED
@@ -65,6 +65,7 @@ module CloudCrowd
|
|
65
65
|
|
66
66
|
# When creating a node, specify the port it should run on.
|
67
67
|
def initialize(options={})
|
68
|
+
super()
|
68
69
|
require 'json'
|
69
70
|
CloudCrowd.identity = :node
|
70
71
|
@central = CloudCrowd.central_server
|
@@ -91,7 +92,7 @@ module CloudCrowd
|
|
91
92
|
@server.daemonize if @daemon
|
92
93
|
trap_signals
|
93
94
|
asset_store
|
94
|
-
@server_thread =
|
95
|
+
@server_thread = CloudCrowd.defer { @server.start }
|
95
96
|
check_in(true)
|
96
97
|
check_in_periodically
|
97
98
|
monitor_system if @max_load || @min_memory
|
@@ -158,7 +159,7 @@ module CloudCrowd
|
|
158
159
|
# average and the amount of free memory remaining. If we transition out of
|
159
160
|
# the overloaded state, let central know.
|
160
161
|
def monitor_system
|
161
|
-
@monitor_thread =
|
162
|
+
@monitor_thread = CloudCrowd.defer do
|
162
163
|
loop do
|
163
164
|
was_overloaded = @overloaded
|
164
165
|
@overloaded = overloaded?
|
@@ -172,7 +173,7 @@ module CloudCrowd
|
|
172
173
|
# will assume that the node has gone down. Checking in will let central know
|
173
174
|
# it's still online.
|
174
175
|
def check_in_periodically
|
175
|
-
@check_in_thread =
|
176
|
+
@check_in_thread = CloudCrowd.defer do
|
176
177
|
loop do
|
177
178
|
sleep CHECK_IN_INTERVAL
|
178
179
|
check_in
|
data/lib/cloud_crowd/server.rb
CHANGED
@@ -71,7 +71,7 @@ module CloudCrowd
|
|
71
71
|
# Distributes all work units to available nodes.
|
72
72
|
post '/jobs' do
|
73
73
|
job = Job.create_from_request(JSON.parse(params[:job]))
|
74
|
-
|
74
|
+
CloudCrowd.defer { WorkUnit.distribute_to_nodes }
|
75
75
|
puts "Job ##{job.id} (#{job.action}) started." unless ENV['RACK_ENV'] == 'test'
|
76
76
|
json job
|
77
77
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloud-crowd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.2.beta
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Ashkenas
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-04-
|
12
|
+
date: 2014-04-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
@@ -288,14 +288,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
288
288
|
version: '0'
|
289
289
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
290
290
|
requirements:
|
291
|
-
- - "
|
291
|
+
- - ">"
|
292
292
|
- !ruby/object:Gem::Version
|
293
|
-
version:
|
293
|
+
version: 1.3.1
|
294
294
|
requirements: []
|
295
295
|
rubyforge_project: cloud-crowd
|
296
|
-
rubygems_version: 2.2.
|
296
|
+
rubygems_version: 2.2.2
|
297
297
|
signing_key:
|
298
298
|
specification_version: 4
|
299
299
|
summary: Parallel Processing for the Rest of Us
|
300
300
|
test_files: []
|
301
|
-
has_rdoc:
|