documentcloud-cloud-crowd 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/actions/graphics_magick.rb +44 -0
- data/bin/crowd +5 -0
- data/cloud-crowd.gemspec +71 -0
- data/config/config.example.ru +17 -0
- data/config/config.example.yml +11 -0
- data/config/database.example.yml +6 -0
- data/lib/cloud-crowd.rb +96 -0
- data/lib/cloud_crowd/action.rb +88 -0
- data/lib/cloud_crowd/app.rb +54 -0
- data/lib/cloud_crowd/asset_store.rb +58 -0
- data/lib/cloud_crowd/command_line.rb +198 -0
- data/lib/cloud_crowd/core_ext.rb +10 -0
- data/lib/cloud_crowd/daemon.rb +48 -0
- data/lib/cloud_crowd/helpers/resources.rb +15 -0
- data/lib/cloud_crowd/helpers/urls.rb +7 -0
- data/lib/cloud_crowd/helpers.rb +8 -0
- data/lib/cloud_crowd/models/job.rb +129 -0
- data/lib/cloud_crowd/models/work_unit.rb +62 -0
- data/lib/cloud_crowd/models.rb +31 -0
- data/lib/cloud_crowd/runner.rb +29 -0
- data/lib/cloud_crowd/schema.rb +34 -0
- data/lib/cloud_crowd/worker.rb +115 -0
- data/test/acceptance/test_failing_work_units.rb +32 -0
- data/test/blueprints.rb +15 -0
- data/test/config/test_config.yml +10 -0
- data/test/config/test_database.yml +6 -0
- data/test/test_helper.rb +18 -0
- data/test/unit/test_job.rb +70 -0
- data/test/unit/test_work_unit.rb +55 -0
- metadata +190 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
# The GraphicsMagick action, dependent on the `gm` command, is able to perform
|
2
|
+
# any number of GraphicsMagick conversions on an image passed in as an input.
|
3
|
+
# The options hash should specify the +name+ for the particular step (which is
|
4
|
+
# appended to the resulting image filename) the +command+ (eg. convert, mogrify),
|
5
|
+
# the +options+ (to the command, eg. -shadow -blur), and the +extension+ which
|
6
|
+
# will determine the resulting image type. Optionally, you may also specify
|
7
|
+
# +input+ as the name of a previous step; doing this will use the result of
|
8
|
+
# that step as the source image, otherwise each step uses the original image
|
9
|
+
# as its source.
|
10
|
+
class GraphicsMagick < CloudCrowd::Action
|
11
|
+
|
12
|
+
# Download the initial image, and run each of the specified GraphicsMagick
|
13
|
+
# commands against it, returning the aggregate output.
|
14
|
+
def run
|
15
|
+
return options['steps'].map {|step| run_step(step) }
|
16
|
+
end
|
17
|
+
|
18
|
+
# Run an individual step (single GraphicsMagick command) in a shell-injection
|
19
|
+
# safe way, uploading the result to the AssetStore, and returning the public
|
20
|
+
# URL as the result.
|
21
|
+
# TODO: +system+ wasn't working, figure out some other way to escape.
|
22
|
+
def run_step(step)
|
23
|
+
name, cmd, opts, ext = step['name'], step['command'], step['options'], step['extension']
|
24
|
+
in_path, out_path = input_path_for(step), output_path_for(step)
|
25
|
+
`gm #{cmd} #{opts} #{in_path} #{out_path}`
|
26
|
+
public_url = save(out_path)
|
27
|
+
{'name' => name, 'url' => public_url}.to_json
|
28
|
+
end
|
29
|
+
|
30
|
+
# Where should the starting image be located?
|
31
|
+
# If you pass in an optional step, returns the path to that step's output
|
32
|
+
# as input for further processing.
|
33
|
+
def input_path_for(step)
|
34
|
+
in_step = step && step['input'] && options['steps'].detect {|s| s['name'] == step['input']}
|
35
|
+
return input_path unless in_step
|
36
|
+
return output_path_for(in_step)
|
37
|
+
end
|
38
|
+
|
39
|
+
# Where should resulting images be saved locally?
|
40
|
+
def output_path_for(step)
|
41
|
+
"#{work_directory}/#{file_name}_#{step['name']}.#{step['extension']}"
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
data/bin/crowd
ADDED
data/cloud-crowd.gemspec
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'cloud-crowd'
|
3
|
+
s.version = '0.0.1'
|
4
|
+
s.date = '2009-08-23'
|
5
|
+
|
6
|
+
s.homepage = "http://documentcloud.org" # wiki page on github?
|
7
|
+
s.summary = "Better living through Map --> Ruby --> Reduce"
|
8
|
+
s.description = <<-EOS
|
9
|
+
The crowd, suddenly there where there was nothing before, is a mysterious and
|
10
|
+
universal phenomenon. A few people may have been standing together -- five, ten
|
11
|
+
or twelve, nor more; nothing has been announced, nothing is expected. Suddenly
|
12
|
+
everywhere is black with people and more come streaming from all sides as though
|
13
|
+
streets had only one direction.
|
14
|
+
EOS
|
15
|
+
|
16
|
+
s.authors = ['Jeremy Ashkenas']
|
17
|
+
s.email = 'jeremy@documentcloud.org'
|
18
|
+
|
19
|
+
s.require_paths = ['lib']
|
20
|
+
s.executables = ['crowd']
|
21
|
+
|
22
|
+
s.post_install_message = "Run `crowd help` for information on using CloudCrowd."
|
23
|
+
s.rubyforge_project = 'cloud-crowd'
|
24
|
+
s.has_rdoc = true
|
25
|
+
|
26
|
+
s.add_dependency 'sinatra', ['>= 0.9.4']
|
27
|
+
s.add_dependency 'activerecord', ['>= 2.3.3']
|
28
|
+
s.add_dependency 'json', ['>= 1.1.7']
|
29
|
+
s.add_dependency 'rest-client', ['>= 1.0.3']
|
30
|
+
s.add_dependency 'right_aws', ['>= 1.10.0']
|
31
|
+
s.add_dependency 'daemons', ['>= 1.0.10']
|
32
|
+
|
33
|
+
if s.respond_to?(:add_development_dependency)
|
34
|
+
s.add_development_dependency 'faker', ['>= 0.3.1']
|
35
|
+
s.add_development_dependency 'thoughtbot-shoulda', ['>= 2.10.2']
|
36
|
+
s.add_development_dependency 'notahat-machinist', ['>= 1.0.3']
|
37
|
+
s.add_development_dependency 'rack-test', ['>= 0.4.1']
|
38
|
+
s.add_development_dependency 'mocha', ['>= 0.9.7']
|
39
|
+
end
|
40
|
+
|
41
|
+
s.files = %w(
|
42
|
+
actions/graphics_magick.rb
|
43
|
+
cloud-crowd.gemspec
|
44
|
+
config/config.example.ru
|
45
|
+
config/config.example.yml
|
46
|
+
config/database.example.yml
|
47
|
+
lib/cloud-crowd.rb
|
48
|
+
lib/cloud_crowd/action.rb
|
49
|
+
lib/cloud_crowd/app.rb
|
50
|
+
lib/cloud_crowd/asset_store.rb
|
51
|
+
lib/cloud_crowd/command_line.rb
|
52
|
+
lib/cloud_crowd/core_ext.rb
|
53
|
+
lib/cloud_crowd/daemon.rb
|
54
|
+
lib/cloud_crowd/helpers/resources.rb
|
55
|
+
lib/cloud_crowd/helpers/urls.rb
|
56
|
+
lib/cloud_crowd/helpers.rb
|
57
|
+
lib/cloud_crowd/models/job.rb
|
58
|
+
lib/cloud_crowd/models/work_unit.rb
|
59
|
+
lib/cloud_crowd/models.rb
|
60
|
+
lib/cloud_crowd/runner.rb
|
61
|
+
lib/cloud_crowd/schema.rb
|
62
|
+
lib/cloud_crowd/worker.rb
|
63
|
+
test/acceptance/test_failing_work_units.rb
|
64
|
+
test/blueprints.rb
|
65
|
+
test/config/test_config.yml
|
66
|
+
test/config/test_database.yml
|
67
|
+
test/test_helper.rb
|
68
|
+
test/unit/test_job.rb
|
69
|
+
test/unit/test_work_unit.rb
|
70
|
+
)
|
71
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# This rackup script can be used to start the central CloudCrowd server
|
4
|
+
# using any Rack-compliant server handler. For example, start up three servers
|
5
|
+
# with a specified port number, using Thin:
|
6
|
+
#
|
7
|
+
# thin start -R config.ru -p 9173 --servers 3
|
8
|
+
|
9
|
+
require 'rubygems'
|
10
|
+
require 'cloud-crowd'
|
11
|
+
|
12
|
+
CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
|
13
|
+
CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
|
14
|
+
|
15
|
+
map '/' do
|
16
|
+
run CloudCrowd::App
|
17
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
:num_workers: 4
|
2
|
+
:default_worker_wait: 1
|
3
|
+
:max_worker_wait: 20
|
4
|
+
:worker_wait_multiplier: 1.3
|
5
|
+
:worker_retry_wait: 5
|
6
|
+
:work_unit_retries: 3
|
7
|
+
|
8
|
+
:central_server: http://localhost:9173
|
9
|
+
:s3_bucket: [your CloudCrowd bucket]
|
10
|
+
:aws_access_key: [your AWS access key]
|
11
|
+
:aws_secret_key: [your AWS secret access key]
|
data/lib/cloud-crowd.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__))
|
2
|
+
|
3
|
+
# Standard Library:
|
4
|
+
require 'tmpdir'
|
5
|
+
require 'erb'
|
6
|
+
|
7
|
+
# Gems:
|
8
|
+
require 'sinatra'
|
9
|
+
require 'activerecord'
|
10
|
+
require 'json'
|
11
|
+
require 'daemons'
|
12
|
+
require 'rest_client'
|
13
|
+
require 'right_aws'
|
14
|
+
|
15
|
+
module CloudCrowd
|
16
|
+
|
17
|
+
class App < Sinatra::Default
|
18
|
+
set :root, File.expand_path(File.dirname(__FILE__) + '/..')
|
19
|
+
end
|
20
|
+
|
21
|
+
# Keep the version in sync with the gemspec.
|
22
|
+
VERSION = '0.0.1'
|
23
|
+
|
24
|
+
# A Job is processing if its WorkUnits in the queue to be handled by workers.
|
25
|
+
PROCESSING = 1
|
26
|
+
|
27
|
+
# A Job has succeeded if all of its WorkUnits have finished successfully.
|
28
|
+
SUCCEEDED = 2
|
29
|
+
|
30
|
+
# A Job has failed if even a single one of its WorkUnits has failed (they may
|
31
|
+
# be attempted multiple times on failure, however).
|
32
|
+
FAILED = 3
|
33
|
+
|
34
|
+
# A Job is splitting if it's in the process of dividing its inputs up into
|
35
|
+
# multiple WorkUnits.
|
36
|
+
SPLITTING = 4
|
37
|
+
|
38
|
+
# A Job is merging if it's busy collecting all of its successful WorkUnits
|
39
|
+
# back together into the final result.
|
40
|
+
MERGING = 5
|
41
|
+
|
42
|
+
# A work unit is considered to be complete if it succeeded or if it failed.
|
43
|
+
COMPLETE = [SUCCEEDED, FAILED]
|
44
|
+
|
45
|
+
# A work unit is considered incomplete if it's being processed, split up or
|
46
|
+
# merged together.
|
47
|
+
INCOMPLETE = [PROCESSING, SPLITTING, MERGING]
|
48
|
+
|
49
|
+
# Mapping of statuses to their display strings.
|
50
|
+
DISPLAY_STATUS_MAP = {
|
51
|
+
1 => 'processing', 2 => 'succeeded', 3 => 'failed', 4 => 'splitting', 5 => 'merging'
|
52
|
+
}
|
53
|
+
|
54
|
+
class << self
|
55
|
+
attr_reader :config
|
56
|
+
|
57
|
+
# Configure CloudCrowd by passing in the path to +config.yml+.
|
58
|
+
def configure(config_path)
|
59
|
+
@config = YAML.load_file(config_path)
|
60
|
+
end
|
61
|
+
|
62
|
+
# Configure the CloudCrowd central database (and connect to it), by passing
|
63
|
+
# in a path to +database.yml+.
|
64
|
+
def configure_database(config_path)
|
65
|
+
configuration = YAML.load_file(config_path)
|
66
|
+
ActiveRecord::Base.establish_connection(configuration)
|
67
|
+
end
|
68
|
+
|
69
|
+
# Return the readable status name of an internal CloudCrowd status number.
|
70
|
+
def display_status(status)
|
71
|
+
DISPLAY_STATUS_MAP[status]
|
72
|
+
end
|
73
|
+
|
74
|
+
# Some workers might not ever need to load all the installed actions,
|
75
|
+
# so we lazy-load them. Think about a variant of this for installing and
|
76
|
+
# loading actions into a running CloudCrowd cluster on the fly.
|
77
|
+
def actions(name)
|
78
|
+
action_class = name.camelize
|
79
|
+
begin
|
80
|
+
Module.const_get(action_class)
|
81
|
+
rescue NameError => e
|
82
|
+
require "#{CloudCrowd::App.root}/actions/#{name}"
|
83
|
+
retry
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
# CloudCrowd:
|
91
|
+
require 'cloud_crowd/core_ext'
|
92
|
+
require 'cloud_crowd/models'
|
93
|
+
require 'cloud_crowd/asset_store'
|
94
|
+
require 'cloud_crowd/action'
|
95
|
+
require 'cloud_crowd/helpers'
|
96
|
+
require 'cloud_crowd/app'
|
@@ -0,0 +1,88 @@
|
|
1
|
+
module CloudCrowd
|
2
|
+
|
3
|
+
# Base CloudCrowd::Action class. Override this with your custom action steps.
|
4
|
+
#
|
5
|
+
# Public API to CloudCrowd::Action subclasses:
|
6
|
+
# +input+, +input_path+, +file_name+, +work_directory+, +options+, +save+
|
7
|
+
#
|
8
|
+
# CloudCrowd::Actions must implement a +process+ method, which must return a
|
9
|
+
# JSON-serializeable object that will be used as the output for the work unit.
|
10
|
+
# Optionally, actions may define +split+ and +merge+ methods to do mapping
|
11
|
+
# and reducing around the input.
|
12
|
+
# +split+ must return an array of inputs.
|
13
|
+
# +merge+ must return the output for the job.
|
14
|
+
# All actions run inside of their individual +work_directory+.
|
15
|
+
class Action
|
16
|
+
|
17
|
+
attr_reader :input, :input_path, :file_name, :options, :work_directory
|
18
|
+
|
19
|
+
# Configuring a new Action sets up all of the read-only variables that
|
20
|
+
# form the bulk of the API for action subclasses. (Paths to read from and
|
21
|
+
# write to).
|
22
|
+
def configure(status, input, options, store)
|
23
|
+
@input, @options, @store = input, options, store
|
24
|
+
@job_id, @work_unit_id = options['job_id'], options['work_unit_id']
|
25
|
+
@work_directory = File.expand_path(File.join(@store.temp_storage_path, storage_prefix))
|
26
|
+
FileUtils.mkdir_p(@work_directory) unless File.exists?(@work_directory)
|
27
|
+
Dir.chdir @work_directory
|
28
|
+
unless status == CloudCrowd::MERGING
|
29
|
+
@input_path = File.join(@work_directory, File.basename(@input))
|
30
|
+
@file_name = File.basename(@input_path, File.extname(@input_path))
|
31
|
+
download(@input, @input_path)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Each CloudCrowd::Action must implement a +process+ method.
|
36
|
+
def process
|
37
|
+
raise NotImplementedError.new("CloudCrowd::Actions must override 'run' with their own processing code.")
|
38
|
+
end
|
39
|
+
|
40
|
+
# Download a file to the specified path using curl.
|
41
|
+
def download(url, path)
|
42
|
+
`curl -s "#{url}" > #{path}`
|
43
|
+
path
|
44
|
+
end
|
45
|
+
|
46
|
+
# Takes a local filesystem path, and returns the public url on S3 where the
|
47
|
+
# file was saved.
|
48
|
+
def save(file_path)
|
49
|
+
save_path = File.join(s3_storage_path, File.basename(file_path))
|
50
|
+
@store.save(file_path, save_path)
|
51
|
+
return @store.url(save_path)
|
52
|
+
end
|
53
|
+
|
54
|
+
# After the Action has finished, we remove the work directory.
|
55
|
+
def cleanup_work_directory
|
56
|
+
Dir.chdir '/'
|
57
|
+
FileUtils.rm_r(@work_directory)
|
58
|
+
end
|
59
|
+
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
# The directory prefix to use for both local and S3 storage.
|
64
|
+
# [action_name]/job_[job_id]/unit_[work_unit_it]
|
65
|
+
def storage_prefix
|
66
|
+
path_parts = []
|
67
|
+
path_parts << underscore(self.class.to_s)
|
68
|
+
path_parts << "job_#{@job_id}"
|
69
|
+
path_parts << "unit_#{@work_unit_id}" if @work_unit_id
|
70
|
+
@storage_prefix ||= File.join(path_parts)
|
71
|
+
end
|
72
|
+
|
73
|
+
def s3_storage_path
|
74
|
+
@s3_storage_path ||= storage_prefix
|
75
|
+
end
|
76
|
+
|
77
|
+
# Pilfered from the ActiveSupport::Inflector.
|
78
|
+
def underscore(word)
|
79
|
+
word.to_s.gsub(/::/, '/').
|
80
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
81
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').
|
82
|
+
tr("-", "_").
|
83
|
+
downcase
|
84
|
+
end
|
85
|
+
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module CloudCrowd
|
2
|
+
|
3
|
+
class App < Sinatra::Default
|
4
|
+
|
5
|
+
# static serves files from /public, methodoverride allows the _method param.
|
6
|
+
enable :static, :methodoverride
|
7
|
+
|
8
|
+
helpers CloudCrowd::Helpers
|
9
|
+
|
10
|
+
# Start a new job. Accepts a JSON representation of the job-to-be.
|
11
|
+
post '/jobs' do
|
12
|
+
Job.create_from_request(JSON.parse(params[:json])).to_json
|
13
|
+
end
|
14
|
+
|
15
|
+
# Check the status of a job, returning the output if finished, and the
|
16
|
+
# number of work units remaining otherwise.
|
17
|
+
get '/jobs/:job_id' do
|
18
|
+
current_job.to_json
|
19
|
+
end
|
20
|
+
|
21
|
+
# Cleans up a Job's saved S3 files. Delete a Job after you're done
|
22
|
+
# downloading the results.
|
23
|
+
delete '/jobs/:job_id' do
|
24
|
+
current_job.cleanup
|
25
|
+
''
|
26
|
+
end
|
27
|
+
|
28
|
+
# Internal method for worker daemons to fetch the work unit at the front
|
29
|
+
# of the queue. Work unit is marked as taken and handed off to the worker.
|
30
|
+
get '/work' do
|
31
|
+
begin
|
32
|
+
unit = WorkUnit.first(:conditions => {:status => CloudCrowd::INCOMPLETE, :taken => false}, :order => "created_at desc")
|
33
|
+
return status(204) && '' unless unit
|
34
|
+
unit.update_attributes(:taken => true)
|
35
|
+
unit.to_json
|
36
|
+
rescue ActiveRecord::StaleObjectError => e
|
37
|
+
return status(204) && ''
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# When workers are done with their unit, either successfully on in failure,
|
42
|
+
# they mark it back on the central server.
|
43
|
+
put '/work/:work_unit_id' do
|
44
|
+
case params[:status]
|
45
|
+
when 'succeeded' then current_work_unit.finish(params[:output], params[:time])
|
46
|
+
when 'failed' then current_work_unit.fail(params[:output], params[:time])
|
47
|
+
else return error(500, "Completing a work unit must specify status.")
|
48
|
+
end
|
49
|
+
return status(204) && ''
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module CloudCrowd
|
2
|
+
|
3
|
+
# The CloudCrowd::AssetStore should provide a common API for stashing and retrieving
|
4
|
+
# assets via URLs, in production this will be S3 but in development it may
|
5
|
+
# be the filesystem or /tmp.
|
6
|
+
class AssetStore
|
7
|
+
include FileUtils
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
mkdir_p temp_storage_path unless File.exists? temp_storage_path
|
11
|
+
end
|
12
|
+
|
13
|
+
# Path to CloudCrowd's temporary local storage.
|
14
|
+
def temp_storage_path
|
15
|
+
"#{Dir.tmpdir}/cloud_crowd_tmp"
|
16
|
+
end
|
17
|
+
|
18
|
+
# Copy a finished file from our local storage to S3.
|
19
|
+
def save(local_path, save_path)
|
20
|
+
ensure_s3_connection
|
21
|
+
@bucket.put(save_path, File.open(local_path), {}, 'public-read')
|
22
|
+
end
|
23
|
+
|
24
|
+
# Cleanup all S3 files for a job that's been completed and retrieved.
|
25
|
+
def cleanup_job(job)
|
26
|
+
ensure_s3_connection
|
27
|
+
@bucket.delete_folder("#{job.action}/job_#{job.id}")
|
28
|
+
end
|
29
|
+
|
30
|
+
# Return the S3 public URL for a finshed file.
|
31
|
+
def url(save_path)
|
32
|
+
@bucket.key(save_path).public_link
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
# Unused for the moment. Think about using the filesystem instead of S3
|
38
|
+
# in development.
|
39
|
+
def save_to_filesystem(local_path, save_path)
|
40
|
+
save_path = File.join("/tmp/cloud_crowd_storage", save_path)
|
41
|
+
save_dir = File.dirname(save_path)
|
42
|
+
mkdir_p save_dir unless File.exists? save_dir
|
43
|
+
cp(local_path, save_path)
|
44
|
+
end
|
45
|
+
|
46
|
+
# Workers, through the course of many WorkUnits, keep around an AssetStore.
|
47
|
+
# Ensure we have a persistent S3 connection after first use.
|
48
|
+
def ensure_s3_connection
|
49
|
+
unless @s3 && @bucket
|
50
|
+
params = {:port => 80, :protocol => 'http'}
|
51
|
+
@s3 = RightAws::S3.new(CloudCrowd.config[:aws_access_key], CloudCrowd.config[:aws_secret_key], params)
|
52
|
+
@bucket = @s3.bucket(CloudCrowd.config[:s3_bucket], true)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -0,0 +1,198 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
module CloudCrowd
|
4
|
+
class CommandLine
|
5
|
+
|
6
|
+
# Configuration files required for the `crowd` command to function.
|
7
|
+
CONFIG_FILES = ['config.yml', 'config.ru', 'database.yml']
|
8
|
+
|
9
|
+
# Path to the Daemons gem script which launches workers.
|
10
|
+
WORKER_RUNNER = File.expand_path("#{File.dirname(__FILE__)}/runner.rb")
|
11
|
+
|
12
|
+
# Command-line banner for the usage message.
|
13
|
+
BANNER = <<-EOS
|
14
|
+
Usage: crowd COMMAND OPTIONS
|
15
|
+
|
16
|
+
COMMANDS:
|
17
|
+
install Install the CloudCrowd configuration files to the specified directory
|
18
|
+
server Start up the central server (requires a database)
|
19
|
+
workers Control worker daemons, use: (start | stop | restart | status | run)
|
20
|
+
console Launch a CloudCrowd console, connected to the central database
|
21
|
+
load_schema Load the schema into the database specified by database.yml
|
22
|
+
|
23
|
+
OPTIONS:
|
24
|
+
EOS
|
25
|
+
|
26
|
+
# Creating a CloudCrowd::CommandLine runs from the contents of ARGV.
|
27
|
+
def initialize
|
28
|
+
parse_options
|
29
|
+
command = ARGV.shift
|
30
|
+
case command
|
31
|
+
when 'console' then run_console
|
32
|
+
when 'server' then run_server
|
33
|
+
when 'workers' then run_workers_command
|
34
|
+
when 'load_schema' then run_load_schema
|
35
|
+
when 'install' then run_install
|
36
|
+
else usage
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
# Spin up an IRB session with the CloudCrowd code loaded in, and a database
|
41
|
+
# connection established. The equivalent of Rails' `script/console`.
|
42
|
+
def run_console
|
43
|
+
require 'irb'
|
44
|
+
require 'irb/completion'
|
45
|
+
load_code
|
46
|
+
connect_to_database
|
47
|
+
IRB.start
|
48
|
+
end
|
49
|
+
|
50
|
+
# Convenience command for quickly spinning up the central server. More
|
51
|
+
# sophisticated deployments, load-balancing across multiple app servers,
|
52
|
+
# should use the config.ru rackup file directly. This method will start
|
53
|
+
# a single Thin server, if Thin is installed, otherwise the rackup defaults
|
54
|
+
# (Mongrel, falling back to WEBrick). The equivalent of Rails' script/server.
|
55
|
+
def run_server
|
56
|
+
ensure_config
|
57
|
+
require 'rubygems'
|
58
|
+
rackup_path = File.expand_path('config.ru')
|
59
|
+
if Gem.available? 'thin'
|
60
|
+
exec "thin -e production -p #{@options[:port]} -R #{rackup_path} start"
|
61
|
+
else
|
62
|
+
exec "rackup -E production -p #{@options[:port]} #{rackup_path}"
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# Load in the database schema to the database specified in 'database.yml'.
|
67
|
+
def run_load_schema
|
68
|
+
load_code
|
69
|
+
connect_to_database
|
70
|
+
require 'cloud_crowd/schema.rb'
|
71
|
+
end
|
72
|
+
|
73
|
+
# Install the required CloudCrowd configuration files into the specified
|
74
|
+
# directory, or the current one.
|
75
|
+
def run_install
|
76
|
+
require 'fileutils'
|
77
|
+
install_path = ARGV.shift || '.'
|
78
|
+
cc_root = File.dirname(__FILE__) + '/../..'
|
79
|
+
FileUtils.mkdir_p install_path unless File.exists?(install_path)
|
80
|
+
install_file "#{cc_root}/config/config.example.yml", "#{install_path}/config.yml"
|
81
|
+
install_file "#{cc_root}/config/config.example.ru", "#{install_path}/config.ru"
|
82
|
+
install_file "#{cc_root}/config/database.example.yml", "#{install_path}/database.yml"
|
83
|
+
install_file "#{cc_root}/actions", "#{install_path}/actions", true
|
84
|
+
end
|
85
|
+
|
86
|
+
# Manipulate worker daemons -- handles all commands that the Daemons gem
|
87
|
+
# provides: start, stop, restart, run, and status.
|
88
|
+
def run_workers_command
|
89
|
+
ensure_config
|
90
|
+
command = ARGV.shift
|
91
|
+
case command
|
92
|
+
when 'start' then start_workers
|
93
|
+
when 'stop' then stop_workers
|
94
|
+
when 'restart' then stop_workers && start_workers
|
95
|
+
when 'run' then run_worker
|
96
|
+
when 'status' then show_worker_status
|
97
|
+
else usage
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# Start up N workers, specified by argument or the number of workers in
|
102
|
+
# config.yml.
|
103
|
+
def start_workers
|
104
|
+
load_code
|
105
|
+
num_workers = @options[:num_workers] || CloudCrowd.config[:num_workers]
|
106
|
+
num_workers.times do
|
107
|
+
`CLOUD_CROWD_CONFIG='#{File.expand_path('config.yml')}' ruby #{WORKER_RUNNER} start`
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
# For debugging, run a single worker in the current process, showing output.
|
112
|
+
def run_worker
|
113
|
+
exec "CLOUD_CROWD_CONFIG='#{File.expand_path('config.yml')}' ruby #{WORKER_RUNNER} run"
|
114
|
+
end
|
115
|
+
|
116
|
+
# Stop all active workers.
|
117
|
+
def stop_workers
|
118
|
+
`ruby #{WORKER_RUNNER} stop`
|
119
|
+
end
|
120
|
+
|
121
|
+
# Display the status of all active workers.
|
122
|
+
def show_worker_status
|
123
|
+
puts `ruby #{WORKER_RUNNER} status`
|
124
|
+
end
|
125
|
+
|
126
|
+
# Print `crowd` usage.
|
127
|
+
def usage
|
128
|
+
puts @option_parser
|
129
|
+
end
|
130
|
+
|
131
|
+
|
132
|
+
private
|
133
|
+
|
134
|
+
# Check for configuration files, either in the current directory, or in
|
135
|
+
# the CLOUD_CROWD_CONFIG environment variable. Exit if they're not found.
|
136
|
+
def ensure_config
|
137
|
+
return if @config_found
|
138
|
+
config_dir = ENV['CLOUD_CROWD_CONFIG'] || '.'
|
139
|
+
Dir.chdir config_dir
|
140
|
+
CONFIG_FILES.all? {|f| File.exists? f } ? @config_dir = true : config_not_found
|
141
|
+
end
|
142
|
+
|
143
|
+
# Parse all options for all actions.
|
144
|
+
# TODO: Think about parsing options per sub-command separately.
|
145
|
+
def parse_options
|
146
|
+
@options = {
|
147
|
+
:db_config => 'database.yml',
|
148
|
+
:port => 9173,
|
149
|
+
}
|
150
|
+
@option_parser = OptionParser.new do |opts|
|
151
|
+
opts.on('-n', '--num-workers NUM', OptionParser::DecimalInteger, 'number of worker processes') do |num|
|
152
|
+
@options[:num_workers] = num
|
153
|
+
end
|
154
|
+
opts.on('-d', '--database-config PATH', 'path to database.yml') do |conf_path|
|
155
|
+
@options[:db_config] = conf_path
|
156
|
+
end
|
157
|
+
opts.on('-p', '--port PORT', 'central server port number') do |port_num|
|
158
|
+
@options[:port] = port_num
|
159
|
+
end
|
160
|
+
opts.on_tail('-v', '--version', 'show version') do
|
161
|
+
load_code
|
162
|
+
puts "CloudCrowd version #{CloudCrowd::VERSION}"
|
163
|
+
exit
|
164
|
+
end
|
165
|
+
end
|
166
|
+
@option_parser.banner = BANNER
|
167
|
+
@option_parser.parse!(ARGV)
|
168
|
+
end
|
169
|
+
|
170
|
+
# Load in the CloudCrowd module code, dependencies, lib files and models.
|
171
|
+
# Not all commands require this.
|
172
|
+
def load_code
|
173
|
+
ensure_config
|
174
|
+
require 'rubygems'
|
175
|
+
require File.dirname(__FILE__) + '/../cloud-crowd'
|
176
|
+
CloudCrowd.configure('config.yml')
|
177
|
+
end
|
178
|
+
|
179
|
+
# Establish a connection to the central server's database. Not all commands
|
180
|
+
# require this.
|
181
|
+
def connect_to_database
|
182
|
+
CloudCrowd.configure_database(@options[:db_config])
|
183
|
+
end
|
184
|
+
|
185
|
+
# Exit with an explanation if the configuration files couldn't be found.
|
186
|
+
def config_not_found
|
187
|
+
puts "`crowd` can't find the CloudCrowd configuration directory. Please either run `crowd` from inside of the configuration directory, or add a CLOUD_CROWD_CONFIG variable to your environment."
|
188
|
+
exit(1)
|
189
|
+
end
|
190
|
+
|
191
|
+
# Install a file and log the installation.
|
192
|
+
def install_file(source, dest, is_dir=false)
|
193
|
+
is_dir ? FileUtils.cp_r(source, dest) : FileUtils.cp(source, dest)
|
194
|
+
puts "installed #{dest}"
|
195
|
+
end
|
196
|
+
|
197
|
+
end
|
198
|
+
end
|