documentcloud-cloud-crowd 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/actions/graphics_magick.rb +6 -5
- data/cloud-crowd.gemspec +6 -4
- data/config/config.example.ru +1 -0
- data/config/config.example.yml +6 -1
- data/lib/cloud-crowd.rb +16 -23
- data/lib/cloud_crowd/action.rb +1 -1
- data/lib/cloud_crowd/app.rb +18 -0
- data/lib/cloud_crowd/asset_store.rb +11 -4
- data/lib/cloud_crowd/command_line.rb +20 -14
- data/lib/cloud_crowd/helpers/authorization.rb +46 -0
- data/lib/cloud_crowd/helpers.rb +2 -1
- data/lib/cloud_crowd/models/job.rb +1 -1
- data/lib/cloud_crowd/models.rb +2 -0
- data/lib/cloud_crowd/runner.rb +7 -8
- data/lib/cloud_crowd/worker.rb +23 -6
- data/test/config/actions/failure_testing.rb +13 -0
- data/test/config/{test_config.yml → config.yml} +0 -0
- data/test/config/{test_database.yml → database.yml} +0 -0
- data/test/test_helper.rb +5 -3
- metadata +7 -6
data/actions/graphics_magick.rb
CHANGED
@@ -11,8 +11,10 @@ class GraphicsMagick < CloudCrowd::Action
|
|
11
11
|
|
12
12
|
# Download the initial image, and run each of the specified GraphicsMagick
|
13
13
|
# commands against it, returning the aggregate output.
|
14
|
-
def
|
15
|
-
|
14
|
+
def process
|
15
|
+
result = {}
|
16
|
+
options['steps'].each {|step| result[step['name']] = run_step(step) }
|
17
|
+
result.to_json
|
16
18
|
end
|
17
19
|
|
18
20
|
# Run an individual step (single GraphicsMagick command) in a shell-injection
|
@@ -20,11 +22,10 @@ class GraphicsMagick < CloudCrowd::Action
|
|
20
22
|
# URL as the result.
|
21
23
|
# TODO: +system+ wasn't working, figure out some other way to escape.
|
22
24
|
def run_step(step)
|
23
|
-
|
25
|
+
cmd, opts = step['command'], step['options']
|
24
26
|
in_path, out_path = input_path_for(step), output_path_for(step)
|
25
27
|
`gm #{cmd} #{opts} #{in_path} #{out_path}`
|
26
|
-
|
27
|
-
{'name' => name, 'url' => public_url}.to_json
|
28
|
+
save(out_path)
|
28
29
|
end
|
29
30
|
|
30
31
|
# Where should the starting image be located?
|
data/cloud-crowd.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'cloud-crowd'
|
3
|
-
s.version = '0.0.
|
3
|
+
s.version = '0.0.2' # Keep version in sync with cloud-cloud.rb
|
4
4
|
s.date = '2009-08-23'
|
5
5
|
|
6
6
|
s.homepage = "http://documentcloud.org" # wiki page on github?
|
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.require_paths = ['lib']
|
20
20
|
s.executables = ['crowd']
|
21
21
|
|
22
|
-
s.post_install_message = "Run `crowd help` for information on using CloudCrowd."
|
22
|
+
# s.post_install_message = "Run `crowd --help` for information on using CloudCrowd."
|
23
23
|
s.rubyforge_project = 'cloud-crowd'
|
24
24
|
s.has_rdoc = true
|
25
25
|
|
@@ -51,6 +51,7 @@ lib/cloud_crowd/asset_store.rb
|
|
51
51
|
lib/cloud_crowd/command_line.rb
|
52
52
|
lib/cloud_crowd/core_ext.rb
|
53
53
|
lib/cloud_crowd/daemon.rb
|
54
|
+
lib/cloud_crowd/helpers/authorization.rb
|
54
55
|
lib/cloud_crowd/helpers/resources.rb
|
55
56
|
lib/cloud_crowd/helpers/urls.rb
|
56
57
|
lib/cloud_crowd/helpers.rb
|
@@ -62,8 +63,9 @@ lib/cloud_crowd/schema.rb
|
|
62
63
|
lib/cloud_crowd/worker.rb
|
63
64
|
test/acceptance/test_failing_work_units.rb
|
64
65
|
test/blueprints.rb
|
65
|
-
test/config/
|
66
|
-
test/config/
|
66
|
+
test/config/config.yml
|
67
|
+
test/config/database.yml
|
68
|
+
test/config/actions/failure_testing.rb
|
67
69
|
test/test_helper.rb
|
68
70
|
test/unit/test_job.rb
|
69
71
|
test/unit/test_work_unit.rb
|
data/config/config.example.ru
CHANGED
data/config/config.example.yml
CHANGED
@@ -6,6 +6,11 @@
|
|
6
6
|
:work_unit_retries: 3
|
7
7
|
|
8
8
|
:central_server: http://localhost:9173
|
9
|
+
:use_http_authentication: no
|
10
|
+
:login: [your login name]
|
11
|
+
:password: [your password]
|
12
|
+
|
13
|
+
:use_s3_authentication: no
|
9
14
|
:s3_bucket: [your CloudCrowd bucket]
|
10
15
|
:aws_access_key: [your AWS access key]
|
11
|
-
:aws_secret_key: [your AWS secret access key]
|
16
|
+
:aws_secret_key: [your AWS secret access key]
|
data/lib/cloud-crowd.rb
CHANGED
@@ -1,25 +1,21 @@
|
|
1
1
|
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
|
2
2
|
|
3
|
-
#
|
4
|
-
require 'tmpdir'
|
5
|
-
require 'erb'
|
6
|
-
|
7
|
-
# Gems:
|
8
|
-
require 'sinatra'
|
9
|
-
require 'activerecord'
|
3
|
+
# Common Gems:
|
10
4
|
require 'json'
|
11
|
-
require 'daemons'
|
12
5
|
require 'rest_client'
|
13
6
|
require 'right_aws'
|
14
7
|
|
8
|
+
# Common CloudCrowd libs:
|
9
|
+
require 'cloud_crowd/core_ext'
|
10
|
+
require 'cloud_crowd/action'
|
11
|
+
|
15
12
|
module CloudCrowd
|
16
13
|
|
17
|
-
|
18
|
-
|
19
|
-
end
|
14
|
+
# Root directory of the CloudCrowd gem.
|
15
|
+
ROOT = File.expand_path(File.dirname(__FILE__) + '/..')
|
20
16
|
|
21
17
|
# Keep the version in sync with the gemspec.
|
22
|
-
VERSION = '0.0.
|
18
|
+
VERSION = '0.0.2'
|
23
19
|
|
24
20
|
# A Job is processing if its WorkUnits in the queue to be handled by workers.
|
25
21
|
PROCESSING = 1
|
@@ -56,6 +52,7 @@ module CloudCrowd
|
|
56
52
|
|
57
53
|
# Configure CloudCrowd by passing in the path to +config.yml+.
|
58
54
|
def configure(config_path)
|
55
|
+
@config_path = File.expand_path(File.dirname(config_path))
|
59
56
|
@config = YAML.load_file(config_path)
|
60
57
|
end
|
61
58
|
|
@@ -77,20 +74,16 @@ module CloudCrowd
|
|
77
74
|
def actions(name)
|
78
75
|
action_class = name.camelize
|
79
76
|
begin
|
77
|
+
raise NameError, "can't find the #{action_class} Action" unless Module.constants.include?(action_class)
|
80
78
|
Module.const_get(action_class)
|
81
79
|
rescue NameError => e
|
82
|
-
|
83
|
-
|
80
|
+
user_action = "#{@config_path}/actions/#{name}"
|
81
|
+
default_action = "#{CloudCrowd::ROOT}/actions/#{name}"
|
82
|
+
require user_action and retry if File.exists? "#{user_action}.rb"
|
83
|
+
require default_action and retry if File.exists? "#{default_action}.rb"
|
84
|
+
raise e
|
84
85
|
end
|
85
86
|
end
|
86
87
|
end
|
87
88
|
|
88
|
-
end
|
89
|
-
|
90
|
-
# CloudCrowd:
|
91
|
-
require 'cloud_crowd/core_ext'
|
92
|
-
require 'cloud_crowd/models'
|
93
|
-
require 'cloud_crowd/asset_store'
|
94
|
-
require 'cloud_crowd/action'
|
95
|
-
require 'cloud_crowd/helpers'
|
96
|
-
require 'cloud_crowd/app'
|
89
|
+
end
|
data/lib/cloud_crowd/action.rb
CHANGED
@@ -34,7 +34,7 @@ module CloudCrowd
|
|
34
34
|
|
35
35
|
# Each CloudCrowd::Action must implement a +process+ method.
|
36
36
|
def process
|
37
|
-
raise NotImplementedError.new("CloudCrowd::Actions must override '
|
37
|
+
raise NotImplementedError.new("CloudCrowd::Actions must override 'process' with their own processing code.")
|
38
38
|
end
|
39
39
|
|
40
40
|
# Download a file to the specified path using curl.
|
data/lib/cloud_crowd/app.rb
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
require 'erb'
|
2
|
+
require 'sinatra'
|
3
|
+
require 'cloud_crowd/models'
|
4
|
+
require 'cloud_crowd/helpers'
|
5
|
+
|
1
6
|
module CloudCrowd
|
2
7
|
|
3
8
|
class App < Sinatra::Default
|
@@ -5,8 +10,15 @@ module CloudCrowd
|
|
5
10
|
# static serves files from /public, methodoverride allows the _method param.
|
6
11
|
enable :static, :methodoverride
|
7
12
|
|
13
|
+
set :root, CloudCrowd::ROOT
|
14
|
+
set :authorization_realm, "CloudCrowd"
|
15
|
+
|
8
16
|
helpers CloudCrowd::Helpers
|
9
17
|
|
18
|
+
before do
|
19
|
+
login_required if CloudCrowd.config[:use_http_authentication]
|
20
|
+
end
|
21
|
+
|
10
22
|
# Start a new job. Accepts a JSON representation of the job-to-be.
|
11
23
|
post '/jobs' do
|
12
24
|
Job.create_from_request(JSON.parse(params[:json])).to_json
|
@@ -49,6 +61,12 @@ module CloudCrowd
|
|
49
61
|
return status(204) && ''
|
50
62
|
end
|
51
63
|
|
64
|
+
# To monitor the central server with Monit, God, Nagios, or another
|
65
|
+
# monitoring tool, you can hit /heartbeat to check.
|
66
|
+
get '/heartbeat' do
|
67
|
+
"buh-bump"
|
68
|
+
end
|
69
|
+
|
52
70
|
end
|
53
71
|
|
54
72
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'tmpdir'
|
2
|
+
|
1
3
|
module CloudCrowd
|
2
4
|
|
3
5
|
# The CloudCrowd::AssetStore should provide a common API for stashing and retrieving
|
@@ -7,6 +9,7 @@ module CloudCrowd
|
|
7
9
|
include FileUtils
|
8
10
|
|
9
11
|
def initialize
|
12
|
+
@use_auth = CloudCrowd.config[:use_s3_authentication]
|
10
13
|
mkdir_p temp_storage_path unless File.exists? temp_storage_path
|
11
14
|
end
|
12
15
|
|
@@ -15,10 +18,12 @@ module CloudCrowd
|
|
15
18
|
"#{Dir.tmpdir}/cloud_crowd_tmp"
|
16
19
|
end
|
17
20
|
|
18
|
-
# Copy a finished file from our local storage to S3.
|
21
|
+
# Copy a finished file from our local storage to S3. Save it publicly if
|
22
|
+
# we're not configured to use S3 authentication.
|
19
23
|
def save(local_path, save_path)
|
20
24
|
ensure_s3_connection
|
21
|
-
@
|
25
|
+
permission = @use_auth ? 'private' : 'public-read'
|
26
|
+
@bucket.put(save_path, File.open(local_path), {}, permission)
|
22
27
|
end
|
23
28
|
|
24
29
|
# Cleanup all S3 files for a job that's been completed and retrieved.
|
@@ -27,9 +32,11 @@ module CloudCrowd
|
|
27
32
|
@bucket.delete_folder("#{job.action}/job_#{job.id}")
|
28
33
|
end
|
29
34
|
|
30
|
-
# Return the S3 public URL for a finshed file.
|
35
|
+
# Return the S3 public URL for a finshed file. Authenticated links expire
|
36
|
+
# after one day by default.
|
31
37
|
def url(save_path)
|
32
|
-
@
|
38
|
+
@use_auth ? @s3.interface.get_link(@bucket, save_path) :
|
39
|
+
@bucket.key(save_path).public_link
|
33
40
|
end
|
34
41
|
|
35
42
|
private
|
@@ -6,8 +6,11 @@ module CloudCrowd
|
|
6
6
|
# Configuration files required for the `crowd` command to function.
|
7
7
|
CONFIG_FILES = ['config.yml', 'config.ru', 'database.yml']
|
8
8
|
|
9
|
+
# Reference the absolute path to the root, because we're about to chdir.
|
10
|
+
CC_ROOT = File.expand_path(File.dirname(__FILE__) + '/../..')
|
11
|
+
|
9
12
|
# Path to the Daemons gem script which launches workers.
|
10
|
-
WORKER_RUNNER = File.expand_path("#{
|
13
|
+
WORKER_RUNNER = File.expand_path("#{CC_ROOT}/lib/cloud_crowd/runner.rb")
|
11
14
|
|
12
15
|
# Command-line banner for the usage message.
|
13
16
|
BANNER = <<-EOS
|
@@ -75,12 +78,11 @@ OPTIONS:
|
|
75
78
|
def run_install
|
76
79
|
require 'fileutils'
|
77
80
|
install_path = ARGV.shift || '.'
|
78
|
-
cc_root = File.dirname(__FILE__) + '/../..'
|
79
81
|
FileUtils.mkdir_p install_path unless File.exists?(install_path)
|
80
|
-
install_file "#{
|
81
|
-
install_file "#{
|
82
|
-
install_file "#{
|
83
|
-
install_file "#{
|
82
|
+
install_file "#{CC_ROOT}/config/config.example.yml", "#{install_path}/config.yml"
|
83
|
+
install_file "#{CC_ROOT}/config/config.example.ru", "#{install_path}/config.ru"
|
84
|
+
install_file "#{CC_ROOT}/config/database.example.yml", "#{install_path}/database.yml"
|
85
|
+
install_file "#{CC_ROOT}/actions", "#{install_path}/actions", true
|
84
86
|
end
|
85
87
|
|
86
88
|
# Manipulate worker daemons -- handles all commands that the Daemons gem
|
@@ -135,8 +137,7 @@ OPTIONS:
|
|
135
137
|
# the CLOUD_CROWD_CONFIG environment variable. Exit if they're not found.
|
136
138
|
def ensure_config
|
137
139
|
return if @config_found
|
138
|
-
|
139
|
-
Dir.chdir config_dir
|
140
|
+
Dir.chdir @options[:config_path]
|
140
141
|
CONFIG_FILES.all? {|f| File.exists? f } ? @config_dir = true : config_not_found
|
141
142
|
end
|
142
143
|
|
@@ -144,16 +145,20 @@ OPTIONS:
|
|
144
145
|
# TODO: Think about parsing options per sub-command separately.
|
145
146
|
def parse_options
|
146
147
|
@options = {
|
147
|
-
:db_config
|
148
|
-
:port
|
148
|
+
:db_config => 'database.yml',
|
149
|
+
:port => 9173,
|
150
|
+
:config_path => ENV['CLOUD_CROWD_CONFIG'] || '.',
|
149
151
|
}
|
150
152
|
@option_parser = OptionParser.new do |opts|
|
151
|
-
opts.on('-
|
152
|
-
@options[:
|
153
|
+
opts.on('-c', '--config PATH', 'path to configuration directory') do |conf_path|
|
154
|
+
@options[:config_path] = conf_path
|
153
155
|
end
|
154
156
|
opts.on('-d', '--database-config PATH', 'path to database.yml') do |conf_path|
|
155
157
|
@options[:db_config] = conf_path
|
156
158
|
end
|
159
|
+
opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
|
160
|
+
@options[:num_workers] = num
|
161
|
+
end
|
157
162
|
opts.on('-p', '--port PORT', 'central server port number') do |port_num|
|
158
163
|
@options[:port] = port_num
|
159
164
|
end
|
@@ -172,19 +177,20 @@ OPTIONS:
|
|
172
177
|
def load_code
|
173
178
|
ensure_config
|
174
179
|
require 'rubygems'
|
175
|
-
require
|
180
|
+
require "#{CC_ROOT}/lib/cloud-crowd"
|
176
181
|
CloudCrowd.configure('config.yml')
|
177
182
|
end
|
178
183
|
|
179
184
|
# Establish a connection to the central server's database. Not all commands
|
180
185
|
# require this.
|
181
186
|
def connect_to_database
|
187
|
+
require 'cloud_crowd/models'
|
182
188
|
CloudCrowd.configure_database(@options[:db_config])
|
183
189
|
end
|
184
190
|
|
185
191
|
# Exit with an explanation if the configuration files couldn't be found.
|
186
192
|
def config_not_found
|
187
|
-
puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or
|
193
|
+
puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or use `crowd -c path/to/config`"
|
188
194
|
exit(1)
|
189
195
|
end
|
190
196
|
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# After sinatra-authorization...
|
2
|
+
|
3
|
+
module CloudCrowd
|
4
|
+
module Helpers
|
5
|
+
module Authorization
|
6
|
+
|
7
|
+
def login_required
|
8
|
+
return if authorized?
|
9
|
+
unauthorized! unless auth.provided?
|
10
|
+
bad_request! unless auth.basic?
|
11
|
+
unauthorized! unless authorize(*auth.credentials)
|
12
|
+
request.env['REMOTE_USER'] = auth.username
|
13
|
+
end
|
14
|
+
|
15
|
+
def authorized?
|
16
|
+
!!request.env['REMOTE_USER']
|
17
|
+
end
|
18
|
+
|
19
|
+
def current_user
|
20
|
+
request.env['REMOTE_USER']
|
21
|
+
end
|
22
|
+
|
23
|
+
def authorize(login, password)
|
24
|
+
return true unless CloudCrowd.config[:use_http_authentication]
|
25
|
+
return CloudCrowd.config[:login] == login &&
|
26
|
+
CloudCrowd.config[:password] == password
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def auth
|
33
|
+
@auth ||= Rack::Auth::Basic::Request.new(request.env)
|
34
|
+
end
|
35
|
+
|
36
|
+
def unauthorized!(realm = CloudCrowd::App.authorization_realm)
|
37
|
+
response['WWW-Authenticate'] = "Basic realm=\"#{realm}\""
|
38
|
+
halt 401, 'Authorization Required'
|
39
|
+
end
|
40
|
+
|
41
|
+
def bad_request!
|
42
|
+
halt 400, 'Bad Request'
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/cloud_crowd/helpers.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
+
require 'cloud_crowd/helpers/authorization'
|
1
2
|
require 'cloud_crowd/helpers/resources'
|
2
3
|
require 'cloud_crowd/helpers/urls'
|
3
4
|
|
4
5
|
module CloudCrowd
|
5
6
|
module Helpers
|
6
|
-
include Resources, Urls #, Rack::Utils
|
7
|
+
include Authorization, Resources, Urls #, Rack::Utils
|
7
8
|
end
|
8
9
|
end
|
@@ -113,7 +113,7 @@ class Job < ActiveRecord::Base
|
|
113
113
|
# WorkUnits, as well as any completed outputs.
|
114
114
|
def to_json(opts={})
|
115
115
|
atts = {'id' => self.id, 'status' => self.display_status, 'work_units_remaining' => self.work_units_remaining}
|
116
|
-
atts.merge!({'
|
116
|
+
atts.merge!({'outputs' => JSON.parse(self.outputs)}) if self.outputs
|
117
117
|
atts.merge!({'time' => self.time}) if self.time
|
118
118
|
atts.to_json
|
119
119
|
end
|
data/lib/cloud_crowd/models.rb
CHANGED
data/lib/cloud_crowd/runner.rb
CHANGED
@@ -2,24 +2,23 @@
|
|
2
2
|
# daemons don't load the entire rails stack, this file functions like a mini
|
3
3
|
# environment.rb, loading all the common gems that we need.
|
4
4
|
|
5
|
-
#
|
6
|
-
|
7
|
-
# Standard Lib and Gems
|
5
|
+
# Standard Libs
|
8
6
|
require 'fileutils'
|
7
|
+
require 'benchmark'
|
8
|
+
require 'socket'
|
9
|
+
|
10
|
+
# Gems
|
9
11
|
require 'rubygems'
|
10
12
|
require 'daemons'
|
11
|
-
require 'socket'
|
12
13
|
require 'yaml'
|
13
|
-
require 'json'
|
14
|
-
require 'rest_client'
|
15
|
-
require 'right_aws'
|
16
14
|
|
17
15
|
FileUtils.mkdir('log') unless File.exists?('log')
|
18
16
|
|
19
17
|
# Daemon/Worker Dependencies.
|
20
18
|
require "#{File.dirname(__FILE__)}/../cloud-crowd"
|
19
|
+
require 'cloud_crowd/asset_store'
|
21
20
|
|
22
|
-
Daemons.run("#{CloudCrowd::
|
21
|
+
Daemons.run("#{CloudCrowd::ROOT}/lib/cloud_crowd/daemon.rb", {
|
23
22
|
:app_name => "cloud_crowd_worker",
|
24
23
|
:dir_mode => :normal,
|
25
24
|
:dir => 'log',
|
data/lib/cloud_crowd/worker.rb
CHANGED
@@ -11,15 +11,17 @@ module CloudCrowd
|
|
11
11
|
# connection to S3. This AssetStore gets passed into each action, for use
|
12
12
|
# as it is run.
|
13
13
|
def initialize
|
14
|
-
@id
|
14
|
+
@id = $$
|
15
15
|
@hostname = Socket.gethostname
|
16
|
-
@store
|
16
|
+
@store = CloudCrowd::AssetStore.new
|
17
|
+
@server = central_server_resource
|
18
|
+
log 'started'
|
17
19
|
end
|
18
20
|
|
19
21
|
# Ask the central server for a new WorkUnit.
|
20
22
|
def fetch_work_unit
|
21
23
|
keep_trying_to "fetch a new work unit" do
|
22
|
-
unit_json =
|
24
|
+
unit_json = @server['/work'].get
|
23
25
|
return unless unit_json # No content means no work for us.
|
24
26
|
@start_time = Time.now
|
25
27
|
parse_work_unit unit_json
|
@@ -31,7 +33,7 @@ module CloudCrowd
|
|
31
33
|
def complete_work_unit(result)
|
32
34
|
keep_trying_to "complete work unit" do
|
33
35
|
data = completion_params.merge({:status => 'succeeded', :output => result})
|
34
|
-
|
36
|
+
@server["/work/#{data[:id]}"].put(data)
|
35
37
|
log "finished #{@action_name} in #{data[:time]} seconds"
|
36
38
|
end
|
37
39
|
end
|
@@ -40,7 +42,7 @@ module CloudCrowd
|
|
40
42
|
def fail_work_unit(exception)
|
41
43
|
keep_trying_to "mark work unit as failed" do
|
42
44
|
data = completion_params.merge({:status => 'failed', :output => exception.message})
|
43
|
-
|
45
|
+
@server["/work/#{data[:id]}"].put(data)
|
44
46
|
log "failed #{@action_name} in #{data[:time]} seconds\n#{exception.message}\n#{exception.backtrace}"
|
45
47
|
end
|
46
48
|
end
|
@@ -63,7 +65,7 @@ module CloudCrowd
|
|
63
65
|
end
|
64
66
|
|
65
67
|
# Executes the current work unit, catching all exceptions as failures.
|
66
|
-
def
|
68
|
+
def run_work_unit
|
67
69
|
begin
|
68
70
|
@action = CloudCrowd.actions(@action_name).new
|
69
71
|
@action.configure(@status, @input, @options, @store)
|
@@ -81,9 +83,24 @@ module CloudCrowd
|
|
81
83
|
end
|
82
84
|
end
|
83
85
|
|
86
|
+
# Wraps +run_work_unit+ to benchmark the execution time, if requested.
|
87
|
+
def run
|
88
|
+
return run_work_unit unless @options['benchmark']
|
89
|
+
status = CloudCrowd.display_status(@status)
|
90
|
+
log("ran #{@action_name}/#{status} in " + Benchmark.measure { run_work_unit }.to_s)
|
91
|
+
end
|
92
|
+
|
84
93
|
|
85
94
|
private
|
86
95
|
|
96
|
+
# Keep an authenticated (if configured to enable authentication) resource
|
97
|
+
# for the central server.
|
98
|
+
def central_server_resource
|
99
|
+
params = [CENTRAL_URL]
|
100
|
+
params += [CloudCrowd.config[:login], CloudCrowd.config[:password]] if CloudCrowd.config[:use_http_authentication]
|
101
|
+
RestClient::Resource.new(*params)
|
102
|
+
end
|
103
|
+
|
87
104
|
# Common parameters to send back to central, regardless of success or failure.
|
88
105
|
def completion_params
|
89
106
|
{:id => @options['work_unit_id'], :time => Time.now - @start_time}
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# Simple Action that fails the work unit until it is just about to exhaust
|
2
|
+
# all of its retries.
|
3
|
+
class FailureTesting < CloudCrowd::Action
|
4
|
+
|
5
|
+
def run
|
6
|
+
if options['attempts'] + 1 >= CloudCrowd.config[:work_unit_retries]
|
7
|
+
return 'made it!'
|
8
|
+
else
|
9
|
+
raise 'hell'
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
File without changes
|
File without changes
|
data/test/test_helper.rb
CHANGED
@@ -2,8 +2,10 @@ require 'rubygems'
|
|
2
2
|
|
3
3
|
here = File.dirname(__FILE__)
|
4
4
|
require File.expand_path(here + "/../lib/cloud-crowd")
|
5
|
-
|
6
|
-
|
5
|
+
require 'cloud_crowd/app'
|
6
|
+
|
7
|
+
CloudCrowd.configure(here + '/config/config.yml')
|
8
|
+
CloudCrowd.configure_database(here + '/config/database.yml')
|
7
9
|
|
8
10
|
require 'faker'
|
9
11
|
require 'sham'
|
@@ -11,7 +13,7 @@ require 'rack/test'
|
|
11
13
|
require 'shoulda/active_record'
|
12
14
|
require 'machinist/active_record'
|
13
15
|
require 'mocha'
|
14
|
-
require "#{CloudCrowd::
|
16
|
+
require "#{CloudCrowd::ROOT}/test/blueprints.rb"
|
15
17
|
|
16
18
|
class Test::Unit::TestCase
|
17
19
|
include CloudCrowd
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: documentcloud-cloud-crowd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Ashkenas
|
@@ -143,6 +143,7 @@ files:
|
|
143
143
|
- lib/cloud_crowd/command_line.rb
|
144
144
|
- lib/cloud_crowd/core_ext.rb
|
145
145
|
- lib/cloud_crowd/daemon.rb
|
146
|
+
- lib/cloud_crowd/helpers/authorization.rb
|
146
147
|
- lib/cloud_crowd/helpers/resources.rb
|
147
148
|
- lib/cloud_crowd/helpers/urls.rb
|
148
149
|
- lib/cloud_crowd/helpers.rb
|
@@ -154,15 +155,15 @@ files:
|
|
154
155
|
- lib/cloud_crowd/worker.rb
|
155
156
|
- test/acceptance/test_failing_work_units.rb
|
156
157
|
- test/blueprints.rb
|
157
|
-
- test/config/
|
158
|
-
- test/config/
|
158
|
+
- test/config/config.yml
|
159
|
+
- test/config/database.yml
|
160
|
+
- test/config/actions/failure_testing.rb
|
159
161
|
- test/test_helper.rb
|
160
162
|
- test/unit/test_job.rb
|
161
163
|
- test/unit/test_work_unit.rb
|
162
164
|
has_rdoc: true
|
163
165
|
homepage: http://documentcloud.org
|
164
|
-
|
165
|
-
post_install_message: Run `crowd help` for information on using CloudCrowd.
|
166
|
+
post_install_message:
|
166
167
|
rdoc_options: []
|
167
168
|
|
168
169
|
require_paths:
|
@@ -182,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
182
183
|
requirements: []
|
183
184
|
|
184
185
|
rubyforge_project: cloud-crowd
|
185
|
-
rubygems_version: 1.
|
186
|
+
rubygems_version: 1.2.0
|
186
187
|
signing_key:
|
187
188
|
specification_version: 2
|
188
189
|
summary: Better living through Map --> Ruby --> Reduce
|