cloud-crowd 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. data/README +16 -16
  2. data/cloud-crowd.gemspec +10 -9
  3. data/config/config.example.ru +8 -2
  4. data/config/config.example.yml +21 -25
  5. data/examples/process_pdfs_example.rb +1 -1
  6. data/examples/word_count_example.rb +1 -0
  7. data/lib/cloud-crowd.rb +47 -28
  8. data/lib/cloud_crowd/action.rb +14 -8
  9. data/lib/cloud_crowd/asset_store.rb +8 -8
  10. data/lib/cloud_crowd/asset_store/filesystem_store.rb +18 -7
  11. data/lib/cloud_crowd/asset_store/s3_store.rb +14 -11
  12. data/lib/cloud_crowd/command_line.rb +24 -58
  13. data/lib/cloud_crowd/exceptions.rb +7 -0
  14. data/lib/cloud_crowd/helpers/authorization.rb +5 -3
  15. data/lib/cloud_crowd/helpers/resources.rb +0 -20
  16. data/lib/cloud_crowd/models.rb +1 -1
  17. data/lib/cloud_crowd/models/job.rb +37 -40
  18. data/lib/cloud_crowd/models/node_record.rb +95 -0
  19. data/lib/cloud_crowd/models/work_unit.rb +87 -33
  20. data/lib/cloud_crowd/node.rb +105 -0
  21. data/lib/cloud_crowd/schema.rb +22 -18
  22. data/lib/cloud_crowd/{app.rb → server.rb} +34 -34
  23. data/lib/cloud_crowd/worker.rb +68 -107
  24. data/public/css/admin_console.css +40 -18
  25. data/public/images/server.png +0 -0
  26. data/public/images/server_busy.png +0 -0
  27. data/public/js/admin_console.js +47 -18
  28. data/test/acceptance/test_failing_work_units.rb +1 -1
  29. data/test/acceptance/{test_app.rb → test_server.rb} +15 -15
  30. data/test/acceptance/test_word_count.rb +3 -9
  31. data/test/blueprints.rb +0 -1
  32. data/test/config/config.ru +1 -1
  33. data/test/config/config.yml +2 -4
  34. data/test/unit/test_action.rb +1 -1
  35. data/test/unit/test_configuration.rb +1 -1
  36. data/test/unit/test_job.rb +3 -0
  37. data/test/unit/test_work_unit.rb +2 -4
  38. data/views/{index.erb → operations_center.erb} +13 -8
  39. metadata +11 -10
  40. data/lib/cloud_crowd/daemon.rb +0 -95
  41. data/lib/cloud_crowd/models/worker_record.rb +0 -61
  42. data/lib/cloud_crowd/runner.rb +0 -15
@@ -4,7 +4,7 @@ require 'test_helper'
4
4
  class FailingWorkUnitsTest < Test::Unit::TestCase
5
5
 
6
6
  should "retry work units when they fail" do
7
- browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::App))
7
+ browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
8
8
 
9
9
  browser.post '/jobs', :job => {
10
10
  'action' => 'failure_testing',
@@ -1,14 +1,14 @@
1
1
  require 'test_helper'
2
2
 
3
- class AppTest < Test::Unit::TestCase
3
+ class ServerTest < Test::Unit::TestCase
4
4
 
5
5
  include Rack::Test::Methods
6
6
 
7
7
  def app
8
- CloudCrowd::App
8
+ CloudCrowd::Server
9
9
  end
10
10
 
11
- context "The CloudCrowd::App (Sinatra)" do
11
+ context "The CloudCrowd::Server (Sinatra)" do
12
12
 
13
13
  setup do
14
14
  CloudCrowd::Job.destroy_all
@@ -17,7 +17,7 @@ class AppTest < Test::Unit::TestCase
17
17
 
18
18
  should "be able to render the Operations Center (GET /)" do
19
19
  get '/'
20
- assert last_response.body.include? '<div id="workers">'
20
+ assert last_response.body.include? '<div id="nodes">'
21
21
  assert last_response.body.include? '<div id="graphs">'
22
22
  end
23
23
 
@@ -29,17 +29,17 @@ class AppTest < Test::Unit::TestCase
29
29
  assert resp['work_unit_count'] == 2
30
30
  end
31
31
 
32
- should "be able to check in a worker daemon, and then check out a work unit" do
33
- put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
34
- assert last_response.successful? && last_response.empty?
35
- post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
36
- checked_out = JSON.parse(last_response.body)
37
- assert checked_out['action'] == 'graphics_magick'
38
- assert checked_out['attempts'] == 0
39
- assert checked_out['status'] == CloudCrowd::PROCESSING
40
- status_check = JSON.parse(get('/worker/101@localhost').body)
41
- assert checked_out == status_check
42
- end
32
+ # should "be able to check in a worker daemon, and then check out a work unit" do
33
+ # put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
34
+ # assert last_response.successful? && last_response.empty?
35
+ # post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
36
+ # checked_out = JSON.parse(last_response.body)
37
+ # assert checked_out['action'] == 'graphics_magick'
38
+ # assert checked_out['attempts'] == 0
39
+ # assert checked_out['status'] == CloudCrowd::PROCESSING
40
+ # status_check = JSON.parse(get('/worker/101@localhost').body)
41
+ # assert checked_out == status_check
42
+ # end
43
43
 
44
44
  should "have a heartbeat" do
45
45
  assert get('/heartbeat').body == 'buh-bump'
@@ -6,7 +6,7 @@ class WordCountTest < Test::Unit::TestCase
6
6
 
7
7
  setup do
8
8
  @asset_store = AssetStore.new
9
- @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::App))
9
+ @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
10
10
  @browser.put('/worker', :name => 'test_worker', :thread_status => 'sleeping')
11
11
  post_job_to_count_words_in_this_file
12
12
  @job_id = JSON.parse(@browser.last_response.body)['id']
@@ -24,15 +24,9 @@ class WordCountTest < Test::Unit::TestCase
24
24
  end
25
25
 
26
26
  should "be able to perform the processing stage of a word_count" do
27
- @browser.post('/work', :worker_name => 'test_worker', :worker_actions => 'word_count')
28
- assert @browser.last_response.ok?
29
- info = JSON.parse(@browser.last_response.body)
30
- assert info['status'] == 1
31
- assert info['action'] == 'word_count'
32
- assert info['input'] == "file://#{File.expand_path(__FILE__)}"
33
- action = CloudCrowd.actions['word_count'].new(info['status'], info['input'], info['options'], @asset_store)
27
+ action = CloudCrowd.actions['word_count'].new(1, "file://#{File.expand_path(__FILE__)}", {}, @asset_store)
34
28
  count = action.process
35
- assert count == 128
29
+ assert count == 104
36
30
  end
37
31
 
38
32
  end
@@ -11,7 +11,6 @@ end
11
11
  CloudCrowd::WorkUnit.blueprint do
12
12
  job { CloudCrowd::Job.make }
13
13
  status { CloudCrowd::PROCESSING }
14
- worker_record_id { nil }
15
14
  input { Sham.url }
16
15
  action { 'graphics_magick' }
17
16
  end
@@ -13,5 +13,5 @@ CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
13
13
  CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
14
14
 
15
15
  map '/' do
16
- run CloudCrowd::App
16
+ run CloudCrowd::Server
17
17
  end
@@ -1,7 +1,5 @@
1
- :num_workers: 4
2
- :min_worker_wait: 1
3
- :max_worker_wait: 20
1
+ :max_workers: 10
4
2
  :work_unit_retries: 3
5
3
 
6
4
  :central_server: http://localhost:9173
7
- :storage: filesystem
5
+ :storage: filesystem
@@ -27,7 +27,7 @@ class ActionTest < Test::Unit::TestCase
27
27
  end
28
28
 
29
29
  should "be able to save (to the filesystem while testing)" do
30
- assert @action.save(@action.input_path) == "file://#{CloudCrowd::AssetStore::LOCAL_STORAGE_PATH}/word_count/job_1/unit_1/test_action.rb"
30
+ assert @action.save(@action.input_path) == "file://#{@store.local_storage_path}/word_count/job_1/unit_1/test_action.rb"
31
31
  end
32
32
 
33
33
  should "be able to clean up after itself" do
@@ -5,7 +5,7 @@ class ConfigurationTest < Test::Unit::TestCase
5
5
  context "CloudCrowd Configuration" do
6
6
 
7
7
  should "have read in config.yml" do
8
- assert CloudCrowd.config[:num_workers] == 4
8
+ assert CloudCrowd.config[:max_workers] == 10
9
9
  assert CloudCrowd.config[:storage] == 'filesystem'
10
10
  end
11
11
 
@@ -26,6 +26,7 @@ class JobTest < Test::Unit::TestCase
26
26
  should "know its completion status" do
27
27
  assert !@job.all_work_units_complete?
28
28
  @unit.update_attributes(:status => CloudCrowd::SUCCEEDED, :output => '{"output":"hello"}')
29
+ @job.check_for_completion
29
30
  assert @job.reload.all_work_units_complete?
30
31
  assert @job.percent_complete == 100
31
32
  assert @job.outputs == "[\"hello\"]"
@@ -60,6 +61,8 @@ class JobTest < Test::Unit::TestCase
60
61
  end
61
62
 
62
63
  should "fire a callback when a job has finished, successfully or not" do
64
+ @job.update_attribute(:callback_url, 'http://example.com/callback')
65
+ CloudCrowd::Job.any_instance.stubs(:fire_callback).returns(true)
63
66
  CloudCrowd::Job.any_instance.expects(:fire_callback)
64
67
  @job.work_units.first.finish('{"output":"output"}', 10)
65
68
  assert @job.all_work_units_complete?
@@ -21,8 +21,6 @@ class WorkUnitTest < Test::Unit::TestCase
21
21
  assert @unit.complete?
22
22
  @unit.status = CloudCrowd::FAILED
23
23
  assert @unit.complete?
24
- @unit.expects :check_for_job_completion
25
- @unit.save
26
24
  end
27
25
 
28
26
  should "have JSON that includes job attributes" do
@@ -34,10 +32,10 @@ class WorkUnitTest < Test::Unit::TestCase
34
32
  end
35
33
 
36
34
  should "be able to retry, on failure" do
37
- @unit.update_attribute :worker_record_id, 10
35
+ @unit.update_attribute :worker_pid, 7337
38
36
  assert @unit.attempts == 0
39
37
  @unit.fail('oops', 10)
40
- assert @unit.worker_record == nil
38
+ assert @unit.worker_pid == nil
41
39
  assert @unit.attempts == 1
42
40
  assert @unit.processing?
43
41
  @unit.fail('oops again', 10)
@@ -36,25 +36,30 @@
36
36
  <div id="sidebar_background"></div>
37
37
  <div id="sidebar_bottom" class="sidebar_back"></div>
38
38
  <div id="sidebar_header" class="small_caps">
39
- <span class="has_workers">Active Worker Daemons</span>
40
- <span class="no_workers">No Workers Running&hellip;</span>
39
+ <span class="has_nodes"></span>
40
+ <span class="no_nodes">No Nodes Online&hellip;</span>
41
41
  </div>
42
- <div id="workers">
43
- <%# Render target for workers. %>
42
+ <div id="nodes">
43
+ <%# Render target for nodes and workers. %>
44
44
  </div>
45
45
  </div>
46
46
 
47
47
  <div id="graphs">
48
48
  <div class="graph_container">
49
- <div class="graph_title">Work Units in Queue:</div>
49
+ <div class="graph_title">Work Units in Queue</div>
50
50
  <div id="work_units_graph" class="graph"></div>
51
51
  </div>
52
52
  <div class="graph_container">
53
- <div class="graph_title">Jobs in Queue:</div>
53
+ <div class="graph_title">Jobs in Queue</div>
54
54
  <div id="jobs_graph" class="graph"></div>
55
55
  </div>
56
56
  <div class="graph_container">
57
- <div class="graph_title">Active Worker Daemons:</div>
57
+ <div class="graph_title">
58
+ Active Nodes
59
+ (<span id="nodes_legend" class="legend_box"></span>)
60
+ and Workers
61
+ (<span id="workers_legend" class="legend_box"></span>)
62
+ </div>
58
63
  <div id="workers_graph" class="graph"></div>
59
64
  </div>
60
65
  </div>
@@ -67,7 +72,7 @@
67
72
  <div id="worker_job_id">job #<span class="job_id"></span> / work unit #<span class="work_unit_id"></span></div>
68
73
  </div>
69
74
  <div id="worker_sleeping">
70
- worker is sleeping&hellip;
75
+ worker exiting&hellip;
71
76
  </div>
72
77
  </div>
73
78
  </div>
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cloud-crowd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Ashkenas
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-09-14 00:00:00 -04:00
12
+ date: 2009-09-17 00:00:00 -04:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -63,14 +63,14 @@ dependencies:
63
63
  version: 1.10.0
64
64
  version:
65
65
  - !ruby/object:Gem::Dependency
66
- name: daemons
66
+ name: thin
67
67
  type: :runtime
68
68
  version_requirement:
69
69
  version_requirements: !ruby/object:Gem::Requirement
70
70
  requirements:
71
71
  - - ">="
72
72
  - !ruby/object:Gem::Version
73
- version: 1.0.10
73
+ version: 1.2.4
74
74
  version:
75
75
  - !ruby/object:Gem::Dependency
76
76
  name: faker
@@ -144,23 +144,22 @@ files:
144
144
  - examples/word_count_example.rb
145
145
  - lib/cloud-crowd.rb
146
146
  - lib/cloud_crowd/action.rb
147
- - lib/cloud_crowd/app.rb
148
147
  - lib/cloud_crowd/asset_store/filesystem_store.rb
149
148
  - lib/cloud_crowd/asset_store/s3_store.rb
150
149
  - lib/cloud_crowd/asset_store.rb
151
150
  - lib/cloud_crowd/command_line.rb
152
- - lib/cloud_crowd/daemon.rb
153
151
  - lib/cloud_crowd/exceptions.rb
154
152
  - lib/cloud_crowd/helpers/authorization.rb
155
153
  - lib/cloud_crowd/helpers/resources.rb
156
154
  - lib/cloud_crowd/helpers.rb
157
155
  - lib/cloud_crowd/inflector.rb
158
156
  - lib/cloud_crowd/models/job.rb
157
+ - lib/cloud_crowd/models/node_record.rb
159
158
  - lib/cloud_crowd/models/work_unit.rb
160
- - lib/cloud_crowd/models/worker_record.rb
161
159
  - lib/cloud_crowd/models.rb
162
- - lib/cloud_crowd/runner.rb
160
+ - lib/cloud_crowd/node.rb
163
161
  - lib/cloud_crowd/schema.rb
162
+ - lib/cloud_crowd/server.rb
164
163
  - lib/cloud_crowd/worker.rb
165
164
  - LICENSE
166
165
  - public/css/admin_console.css
@@ -171,6 +170,8 @@ files:
171
170
  - public/images/header_back.png
172
171
  - public/images/logo.png
173
172
  - public/images/queue_fill.png
173
+ - public/images/server.png
174
+ - public/images/server_busy.png
174
175
  - public/images/server_error.png
175
176
  - public/images/sidebar_bottom.png
176
177
  - public/images/sidebar_top.png
@@ -181,7 +182,7 @@ files:
181
182
  - public/js/flot.js
182
183
  - public/js/jquery.js
183
184
  - README
184
- - test/acceptance/test_app.rb
185
+ - test/acceptance/test_server.rb
185
186
  - test/acceptance/test_failing_work_units.rb
186
187
  - test/acceptance/test_word_count.rb
187
188
  - test/blueprints.rb
@@ -194,7 +195,7 @@ files:
194
195
  - test/unit/test_configuration.rb
195
196
  - test/unit/test_job.rb
196
197
  - test/unit/test_work_unit.rb
197
- - views/index.erb
198
+ - views/operations_center.erb
198
199
  has_rdoc: true
199
200
  homepage: http://wiki.github.com/documentcloud/cloud-crowd
200
201
  licenses: []
@@ -1,95 +0,0 @@
1
- CloudCrowd.configure(ENV['CLOUD_CROWD_CONFIG'])
2
-
3
- module CloudCrowd
4
-
5
- # A CloudCrowd::Daemon, started by the Daemons gem, runs a CloudCrowd::Worker in
6
- # a loop, continually fetching and processing WorkUnits from the central
7
- # server.
8
- #
9
- # The Daemon backs off and pings the central server less frequently
10
- # when there isn't any work to be done, and speeds back up when there is.
11
- #
12
- # The `crowd` command responds to all the usual methods that the Daemons gem
13
- # supports.
14
- class Daemon
15
-
16
- # The back-off factor used to slow down requests for new work units
17
- # when the queue is empty.
18
- WAIT_MULTIPLIER = 1.5
19
-
20
- MIN_WAIT = CloudCrowd.config[:min_worker_wait]
21
- MAX_WAIT = CloudCrowd.config[:max_worker_wait]
22
-
23
- def initialize
24
- @wait_time = MIN_WAIT
25
- @worker = Worker.new
26
- Signal.trap('INT') { kill_worker_and_exit }
27
- Signal.trap('KILL') { kill_worker_and_exit }
28
- Signal.trap('TERM') { kill_worker_and_exit }
29
- end
30
-
31
- # Spin up our worker and monitoring threads. The monitor's the boss, and
32
- # will feel no compunction in killing the worker thread if necessary.
33
- # Check in before starting up. If check in fails, there's no sense in going.
34
- def run
35
- @worker.check_in('starting')
36
- @work_thread = run_worker
37
- @monitor_thread = run_monitor
38
- @monitor_thread.join
39
- end
40
-
41
-
42
- private
43
-
44
- # Loop forever, fetching WorkUnits and processing them.
45
- def run_worker
46
- Thread.new do
47
- loop do
48
- @worker.fetch_work_unit
49
- if @worker.has_work?
50
- @wait_time = MIN_WAIT
51
- while @worker.has_work?
52
- @worker.run
53
- sleep 0.01 # So as to listen for incoming signals.
54
- end
55
- else
56
- @wait_time = [@wait_time * WAIT_MULTIPLIER, MAX_WAIT].min
57
- sleep @wait_time
58
- end
59
- end
60
- end
61
- end
62
-
63
- # Checks in to let the central server know it's still alive every
64
- # CHECK_IN_INTERVAL seconds. Restarts the work_thread if it has died.
65
- def run_monitor
66
- Thread.new do
67
- sleep Worker::CHECK_IN_INTERVAL
68
- loop do
69
- @work_thread = run_monitor unless @work_thread.alive? || @exit_started
70
- @worker.check_in(@work_thread.status)
71
- sleep Worker::CHECK_IN_INTERVAL
72
- end
73
- end
74
- end
75
-
76
- def running?
77
- @work_thread.alive? || @monitor_thread.alive?
78
- end
79
-
80
- # At exit, kill the worker thread, gently at first, then forcefully.
81
- def kill_worker_and_exit
82
- @worker.check_out
83
- @exit_started = Time.now
84
- @work_thread.kill && @monitor_thread.kill
85
- sleep 0.3 while running? && Time.now - @exit_started < WORKER_EXIT_WAIT
86
- return Process.exit unless running?
87
- @work_thread.kill! && @monitor_thread.kill!
88
- Process.exit
89
- end
90
-
91
- end
92
-
93
- end
94
-
95
- CloudCrowd::Daemon.new.run
@@ -1,61 +0,0 @@
1
- module CloudCrowd
2
-
3
- # A WorkerRecord is a recording of an active worker daemon running remotely.
4
- # Every time it checks in, we keep track of its status. The attributes shown
5
- # here may lag their actual values by up to Worker::CHECK_IN_INTERVAL seconds.
6
- class WorkerRecord < ActiveRecord::Base
7
-
8
- EXPIRES_AFTER = 2 * Worker::CHECK_IN_INTERVAL
9
-
10
- has_one :work_unit
11
-
12
- validates_presence_of :name, :thread_status
13
-
14
- before_destroy :clear_work_units
15
-
16
- named_scope :alive, lambda { {:conditions => ['updated_at > ?', Time.now - EXPIRES_AFTER]} }
17
- named_scope :dead, lambda { {:conditions => ['updated_at <= ?', Time.now - EXPIRES_AFTER]} }
18
-
19
- # Save a Worker's current status to the database.
20
- def self.check_in(params)
21
- attrs = {:thread_status => params[:thread_status], :updated_at => Time.now}
22
- self.find_or_create_by_name(params[:name]).update_attributes!(attrs)
23
- end
24
-
25
- # Remove a terminated Worker's record from the database.
26
- def self.check_out(params)
27
- self.find_by_name(params[:name]).destroy
28
- end
29
-
30
- # We consider the worker to be alive if it's checked in more recently
31
- # than twice the expected interval ago.
32
- def alive?
33
- updated_at > Time.now - EXPIRES_AFTER
34
- end
35
-
36
- # Derive the Worker's PID on the remote machine from the name.
37
- def pid
38
- @pid ||= self.name.split('@').first
39
- end
40
-
41
- # Derive the hostname from the Worker's name.
42
- def hostname
43
- @hostname ||= self.name.split('@').last
44
- end
45
-
46
- def to_json(opts={})
47
- {
48
- 'name' => name,
49
- 'status' => work_unit && work_unit.display_status,
50
- }.to_json
51
- end
52
-
53
-
54
- private
55
-
56
- def clear_work_units
57
- WorkUnit.update_all('worker_record_id = null', "worker_record_id = #{id}")
58
- end
59
-
60
- end
61
- end