cloud-crowd 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/cloud-crowd.gemspec +3 -3
- data/lib/cloud-crowd.rb +10 -3
- data/lib/cloud_crowd/models/node_record.rb +1 -1
- data/lib/cloud_crowd/models/work_unit.rb +2 -1
- data/lib/cloud_crowd/server.rb +2 -2
- data/lib/cloud_crowd/worker.rb +2 -1
- metadata +30 -4
data/cloud-crowd.gemspec
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'cloud-crowd'
|
3
|
-
s.version = '0.5.
|
4
|
-
s.date = '2010-
|
3
|
+
s.version = '0.5.2' # Keep version in sync with cloud-cloud.rb
|
4
|
+
s.date = '2010-08-03'
|
5
5
|
|
6
6
|
s.homepage = "http://wiki.github.com/documentcloud/cloud-crowd"
|
7
7
|
s.summary = "Parallel Processing for the Rest of Us"
|
@@ -113,4 +113,4 @@ test/unit/test_worker.rb
|
|
113
113
|
test/unit/test_work_unit.rb
|
114
114
|
views/operations_center.erb
|
115
115
|
)
|
116
|
-
end
|
116
|
+
end
|
data/lib/cloud-crowd.rb
CHANGED
@@ -45,7 +45,7 @@ module CloudCrowd
|
|
45
45
|
autoload :WorkUnit, 'cloud_crowd/models'
|
46
46
|
|
47
47
|
# Keep this version in sync with the gemspec.
|
48
|
-
VERSION = '0.5.
|
48
|
+
VERSION = '0.5.2'
|
49
49
|
|
50
50
|
# Increment the schema version when there's a backwards incompatible change.
|
51
51
|
SCHEMA_VERSION = 4
|
@@ -59,6 +59,9 @@ module CloudCrowd
|
|
59
59
|
# Default folder to contain the pids of daemonized servers and nodes.
|
60
60
|
PID_PATH = 'tmp/pids'
|
61
61
|
|
62
|
+
# Minimum number of attempts per work unit.
|
63
|
+
MIN_RETRIES = 1
|
64
|
+
|
62
65
|
# A Job is processing if its WorkUnits are in the queue to be handled by nodes.
|
63
66
|
PROCESSING = 1
|
64
67
|
|
@@ -94,6 +97,7 @@ module CloudCrowd
|
|
94
97
|
def configure(config_path)
|
95
98
|
@config_path = File.expand_path(File.dirname(config_path))
|
96
99
|
@config = YAML.load(ERB.new(File.read(config_path)).result)
|
100
|
+
@config[:work_unit_retries] ||= MIN_RETRIES
|
97
101
|
end
|
98
102
|
|
99
103
|
# Configure the CloudCrowd central database (and connect to it), by passing
|
@@ -133,7 +137,10 @@ module CloudCrowd
|
|
133
137
|
# a connection, and a timeout of 30 to finish reading it.
|
134
138
|
def client_options
|
135
139
|
return @client_options if @client_options
|
136
|
-
@client_options = {
|
140
|
+
@client_options = {
|
141
|
+
:timeout => (self.server? ? config[:node_timeout] : config[:server_timeout]) || 30,
|
142
|
+
:open_timeout => config[:open_timeout] || 5
|
143
|
+
}
|
137
144
|
if CloudCrowd.config[:http_authentication]
|
138
145
|
@client_options[:user] = CloudCrowd.config[:login]
|
139
146
|
@client_options[:password] = CloudCrowd.config[:password]
|
@@ -186,4 +193,4 @@ module CloudCrowd
|
|
186
193
|
|
187
194
|
end
|
188
195
|
|
189
|
-
end
|
196
|
+
end
|
@@ -47,7 +47,7 @@ module CloudCrowd
|
|
47
47
|
rescue RestClient::RequestFailed => e
|
48
48
|
raise e unless e.http_code == 503 && e.http_body == Node::OVERLOADED_MESSAGE
|
49
49
|
update_attribute(:busy, true) && false
|
50
|
-
rescue RestClient::Exception, Errno::ECONNREFUSED, Timeout::Error
|
50
|
+
rescue RestClient::Exception, Errno::ECONNREFUSED, Timeout::Error, RestClient::RequestTimeout, Errno::ECONNRESET
|
51
51
|
# Couldn't post to node, assume it's gone away.
|
52
52
|
destroy && false
|
53
53
|
end
|
@@ -41,6 +41,7 @@ module CloudCrowd
|
|
41
41
|
reservation = nil
|
42
42
|
filter = {}
|
43
43
|
loop do
|
44
|
+
WorkUnit.cancel_reservations(reservation) if reservation
|
44
45
|
return unless reservation = WorkUnit.reserve_available(:limit => RESERVATION_LIMIT, :conditions => filter)
|
45
46
|
work_units = WorkUnit.reserved(reservation)
|
46
47
|
available_nodes = NodeRecord.available
|
@@ -69,7 +70,7 @@ module CloudCrowd
|
|
69
70
|
# were none available.
|
70
71
|
def self.reserve_available(options={})
|
71
72
|
reservation = ActiveSupport::SecureRandom.random_number(MAX_RESERVATION)
|
72
|
-
any = WorkUnit.available.update_all("reservation = #{reservation}",
|
73
|
+
any = WorkUnit.available.update_all("reservation = #{reservation}", options[:conditions], options) > 0
|
73
74
|
any && reservation
|
74
75
|
end
|
75
76
|
|
data/lib/cloud_crowd/server.rb
CHANGED
@@ -70,7 +70,7 @@ module CloudCrowd
|
|
70
70
|
# Distributes all work units to available nodes.
|
71
71
|
post '/jobs' do
|
72
72
|
job = Job.create_from_request(JSON.parse(params[:job]))
|
73
|
-
WorkUnit.distribute_to_nodes
|
73
|
+
Thread.new { WorkUnit.distribute_to_nodes }
|
74
74
|
puts "Job ##{job.id} (#{job.action}) started." unless ENV['RACK_ENV'] == 'test'
|
75
75
|
json job
|
76
76
|
end
|
@@ -128,4 +128,4 @@ module CloudCrowd
|
|
128
128
|
|
129
129
|
end
|
130
130
|
|
131
|
-
end
|
131
|
+
end
|
data/lib/cloud_crowd/worker.rb
CHANGED
@@ -24,6 +24,7 @@ module CloudCrowd
|
|
24
24
|
@unit = unit
|
25
25
|
@status = @unit['status']
|
26
26
|
@retry_wait = RETRY_WAIT
|
27
|
+
$0 = "#{unit['action']} (#{unit['id']}) [cloud-crowd-worker]"
|
27
28
|
end
|
28
29
|
|
29
30
|
# Return output to the central server, marking the WorkUnit done.
|
@@ -146,4 +147,4 @@ module CloudCrowd
|
|
146
147
|
|
147
148
|
end
|
148
149
|
|
149
|
-
end
|
150
|
+
end
|
metadata
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloud-crowd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
+
hash: 15
|
4
5
|
prerelease: false
|
5
6
|
segments:
|
6
7
|
- 0
|
7
8
|
- 5
|
8
|
-
-
|
9
|
-
version: 0.5.
|
9
|
+
- 2
|
10
|
+
version: 0.5.2
|
10
11
|
platform: ruby
|
11
12
|
authors:
|
12
13
|
- Jeremy Ashkenas
|
@@ -14,16 +15,18 @@ autorequire:
|
|
14
15
|
bindir: bin
|
15
16
|
cert_chain: []
|
16
17
|
|
17
|
-
date: 2010-
|
18
|
+
date: 2010-08-03 00:00:00 -04:00
|
18
19
|
default_executable:
|
19
20
|
dependencies:
|
20
21
|
- !ruby/object:Gem::Dependency
|
21
22
|
name: sinatra
|
22
23
|
prerelease: false
|
23
24
|
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
24
26
|
requirements:
|
25
27
|
- - ~>
|
26
28
|
- !ruby/object:Gem::Version
|
29
|
+
hash: 25
|
27
30
|
segments:
|
28
31
|
- 0
|
29
32
|
- 9
|
@@ -34,9 +37,11 @@ dependencies:
|
|
34
37
|
name: activerecord
|
35
38
|
prerelease: false
|
36
39
|
requirement: &id002 !ruby/object:Gem::Requirement
|
40
|
+
none: false
|
37
41
|
requirements:
|
38
42
|
- - ~>
|
39
43
|
- !ruby/object:Gem::Version
|
44
|
+
hash: 5
|
40
45
|
segments:
|
41
46
|
- 2
|
42
47
|
- 3
|
@@ -47,9 +52,11 @@ dependencies:
|
|
47
52
|
name: json
|
48
53
|
prerelease: false
|
49
54
|
requirement: &id003 !ruby/object:Gem::Requirement
|
55
|
+
none: false
|
50
56
|
requirements:
|
51
57
|
- - ">="
|
52
58
|
- !ruby/object:Gem::Version
|
59
|
+
hash: 29
|
53
60
|
segments:
|
54
61
|
- 1
|
55
62
|
- 1
|
@@ -61,9 +68,11 @@ dependencies:
|
|
61
68
|
name: rest-client
|
62
69
|
prerelease: false
|
63
70
|
requirement: &id004 !ruby/object:Gem::Requirement
|
71
|
+
none: false
|
64
72
|
requirements:
|
65
73
|
- - ">="
|
66
74
|
- !ruby/object:Gem::Version
|
75
|
+
hash: 7
|
67
76
|
segments:
|
68
77
|
- 1
|
69
78
|
- 4
|
@@ -74,9 +83,11 @@ dependencies:
|
|
74
83
|
name: thin
|
75
84
|
prerelease: false
|
76
85
|
requirement: &id005 !ruby/object:Gem::Requirement
|
86
|
+
none: false
|
77
87
|
requirements:
|
78
88
|
- - ">="
|
79
89
|
- !ruby/object:Gem::Version
|
90
|
+
hash: 23
|
80
91
|
segments:
|
81
92
|
- 1
|
82
93
|
- 2
|
@@ -88,9 +99,11 @@ dependencies:
|
|
88
99
|
name: faker
|
89
100
|
prerelease: false
|
90
101
|
requirement: &id006 !ruby/object:Gem::Requirement
|
102
|
+
none: false
|
91
103
|
requirements:
|
92
104
|
- - ">="
|
93
105
|
- !ruby/object:Gem::Version
|
106
|
+
hash: 17
|
94
107
|
segments:
|
95
108
|
- 0
|
96
109
|
- 3
|
@@ -102,9 +115,11 @@ dependencies:
|
|
102
115
|
name: thoughtbot-shoulda
|
103
116
|
prerelease: false
|
104
117
|
requirement: &id007 !ruby/object:Gem::Requirement
|
118
|
+
none: false
|
105
119
|
requirements:
|
106
120
|
- - ">="
|
107
121
|
- !ruby/object:Gem::Version
|
122
|
+
hash: 35
|
108
123
|
segments:
|
109
124
|
- 2
|
110
125
|
- 10
|
@@ -116,9 +131,11 @@ dependencies:
|
|
116
131
|
name: notahat-machinist
|
117
132
|
prerelease: false
|
118
133
|
requirement: &id008 !ruby/object:Gem::Requirement
|
134
|
+
none: false
|
119
135
|
requirements:
|
120
136
|
- - ">="
|
121
137
|
- !ruby/object:Gem::Version
|
138
|
+
hash: 17
|
122
139
|
segments:
|
123
140
|
- 1
|
124
141
|
- 0
|
@@ -130,9 +147,11 @@ dependencies:
|
|
130
147
|
name: rack-test
|
131
148
|
prerelease: false
|
132
149
|
requirement: &id009 !ruby/object:Gem::Requirement
|
150
|
+
none: false
|
133
151
|
requirements:
|
134
152
|
- - ">="
|
135
153
|
- !ruby/object:Gem::Version
|
154
|
+
hash: 13
|
136
155
|
segments:
|
137
156
|
- 0
|
138
157
|
- 4
|
@@ -144,9 +163,11 @@ dependencies:
|
|
144
163
|
name: mocha
|
145
164
|
prerelease: false
|
146
165
|
requirement: &id010 !ruby/object:Gem::Requirement
|
166
|
+
none: false
|
147
167
|
requirements:
|
148
168
|
- - ">="
|
149
169
|
- !ruby/object:Gem::Version
|
170
|
+
hash: 53
|
150
171
|
segments:
|
151
172
|
- 0
|
152
173
|
- 9
|
@@ -233,6 +254,7 @@ files:
|
|
233
254
|
- test/unit/test_worker.rb
|
234
255
|
- test/unit/test_work_unit.rb
|
235
256
|
- views/operations_center.erb
|
257
|
+
- bin/crowd
|
236
258
|
has_rdoc: true
|
237
259
|
homepage: http://wiki.github.com/documentcloud/cloud-crowd
|
238
260
|
licenses: []
|
@@ -249,23 +271,27 @@ rdoc_options:
|
|
249
271
|
require_paths:
|
250
272
|
- lib
|
251
273
|
required_ruby_version: !ruby/object:Gem::Requirement
|
274
|
+
none: false
|
252
275
|
requirements:
|
253
276
|
- - ">="
|
254
277
|
- !ruby/object:Gem::Version
|
278
|
+
hash: 3
|
255
279
|
segments:
|
256
280
|
- 0
|
257
281
|
version: "0"
|
258
282
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
283
|
+
none: false
|
259
284
|
requirements:
|
260
285
|
- - ">="
|
261
286
|
- !ruby/object:Gem::Version
|
287
|
+
hash: 3
|
262
288
|
segments:
|
263
289
|
- 0
|
264
290
|
version: "0"
|
265
291
|
requirements: []
|
266
292
|
|
267
293
|
rubyforge_project: cloud-crowd
|
268
|
-
rubygems_version: 1.3.
|
294
|
+
rubygems_version: 1.3.7
|
269
295
|
signing_key:
|
270
296
|
specification_version: 3
|
271
297
|
summary: Parallel Processing for the Rest of Us
|