documentcloud-cloud-crowd 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. data/README +8 -8
  2. data/cloud-crowd.gemspec +8 -8
  3. data/config/config.example.ru +8 -2
  4. data/config/config.example.yml +6 -15
  5. data/examples/process_pdfs_example.rb +1 -1
  6. data/examples/word_count_example.rb +1 -0
  7. data/lib/cloud-crowd.rb +6 -5
  8. data/lib/cloud_crowd/action.rb +11 -7
  9. data/lib/cloud_crowd/asset_store/filesystem_store.rb +5 -0
  10. data/lib/cloud_crowd/asset_store/s3_store.rb +7 -3
  11. data/lib/cloud_crowd/asset_store.rb +1 -1
  12. data/lib/cloud_crowd/command_line.rb +14 -53
  13. data/lib/cloud_crowd/exceptions.rb +4 -0
  14. data/lib/cloud_crowd/helpers/authorization.rb +2 -2
  15. data/lib/cloud_crowd/helpers/resources.rb +0 -20
  16. data/lib/cloud_crowd/models/job.rb +25 -26
  17. data/lib/cloud_crowd/models/node_record.rb +81 -0
  18. data/lib/cloud_crowd/models/work_unit.rb +70 -30
  19. data/lib/cloud_crowd/models.rb +1 -1
  20. data/lib/cloud_crowd/node.rb +87 -0
  21. data/lib/cloud_crowd/schema.rb +19 -16
  22. data/lib/cloud_crowd/{app.rb → server.rb} +25 -30
  23. data/lib/cloud_crowd/worker.rb +50 -74
  24. data/public/css/admin_console.css +26 -14
  25. data/public/images/server.png +0 -0
  26. data/public/js/admin_console.js +45 -18
  27. data/test/acceptance/test_failing_work_units.rb +1 -1
  28. data/test/acceptance/{test_app.rb → test_server.rb} +15 -15
  29. data/test/acceptance/test_word_count.rb +3 -9
  30. data/test/blueprints.rb +0 -1
  31. data/test/config/config.ru +1 -1
  32. data/test/config/config.yml +1 -3
  33. data/test/unit/test_configuration.rb +1 -1
  34. data/test/unit/test_job.rb +1 -0
  35. data/test/unit/test_work_unit.rb +2 -4
  36. data/views/index.erb +13 -8
  37. metadata +9 -9
  38. data/lib/cloud_crowd/daemon.rb +0 -95
  39. data/lib/cloud_crowd/models/worker_record.rb +0 -61
  40. data/lib/cloud_crowd/runner.rb +0 -15
@@ -134,40 +134,43 @@ body {
134
134
  }
135
135
  #sidebar_header {
136
136
  position: absolute;
137
+ width: 250px;
137
138
  top: 5px; left: 8px;
138
139
  color: #404040;
139
140
  text-shadow: 0px 1px 1px #eee;
140
141
  }
141
- #sidebar_header.no_workers .no_workers,
142
- #sidebar_header .has_workers {
142
+ #sidebar_header.no_nodes .no_nodes,
143
+ #sidebar_header .has_nodes {
143
144
  display: block;
144
145
  }
145
- #sidebar_header .no_workers,
146
- #sidebar_header.no_workers .has_workers {
146
+ #sidebar_header .no_nodes,
147
+ #sidebar_header.no_nodes .has_nodes {
147
148
  display: none;
148
149
  }
149
- #workers {
150
+ #nodes {
150
151
  position: absolute;
151
152
  padding: 2px 0;
152
153
  top: 21px; left: 0; bottom: 21px;
153
154
  width: 298px;
154
155
  overflow-y: auto; overflow-x: hidden;
155
156
  }
156
- #workers .worker {
157
+ #nodes .node, #nodes .worker {
157
158
  border: 1px solid transparent;
158
159
  margin: 1px 7px;
159
160
  padding-left: 18px;
160
- font-size: 11px;
161
- line-height: 22px;
162
- background: url(/images/bullet_white.png) no-repeat left center;
163
- cursor: pointer;
164
161
  }
165
- #workers .worker.processing,
166
- #workers .worker.splitting,
167
- #workers .worker.merging {
162
+ #nodes .node {
163
+ font-size: 11px;
164
+ line-height: 22px;
165
+ background: url(/images/server.png) no-repeat left center;
166
+ }
167
+ #nodes .worker {
168
+ font-size: 10px;
169
+ line-height: 18px;
170
+ cursor: pointer;
168
171
  background: url(/images/bullet_green.png) no-repeat left center;
169
172
  }
170
- #workers .worker:hover {
173
+ #nodes .worker:hover {
171
174
  border: 1px solid #aaa;
172
175
  border-radius: 4px; -moz-border-radius: 4px; -webkit-border-radius: 4px;
173
176
  background-color: #ccc;
@@ -216,6 +219,15 @@ body {
216
219
  text-shadow: 0px 1px 1px #eee;
217
220
  margin-bottom: 10px;
218
221
  }
222
+ .legend_box {
223
+ display: inline-block;
224
+ width: 10px; height: 10px;
225
+ border: 1px solid #bbb;
226
+ position: relative;
227
+ top: 1px;
228
+ margin: 0 1px;
229
+ background-color: #a1003d;
230
+ }
219
231
  .graph {
220
232
  height: 150px;
221
233
  }
Binary file
@@ -22,26 +22,30 @@ window.Console = {
22
22
 
23
23
  // All options for drawing the system graphs.
24
24
  GRAPH_OPTIONS : {
25
- xaxis : {mode : 'time', timeformat : '%M:%S'},
26
- yaxis : {tickDecimals : 0},
27
- legend : {show : false},
28
- grid : {backgroundColor : '#7f7f7f', color : '#555', tickColor : '#666', borderWidth : 2}
25
+ xaxis : {mode : 'time', timeformat : '%M:%S'},
26
+ yaxis : {tickDecimals : 0},
27
+ legend : {show : false},
28
+ grid : {backgroundColor : '#7f7f7f', color : '#555', tickColor : '#666', borderWidth : 2}
29
29
  },
30
30
  JOBS_COLOR : '#db3a0f',
31
- WORKERS_COLOR : '#a1003d',
31
+ NODES_COLOR : '#1870ab',
32
+ WORKERS_COLOR : '#45a4e5',
32
33
  WORK_UNITS_COLOR : '#ffba14',
33
34
 
34
35
  // Starting the console begins polling the server.
35
36
  initialize : function() {
36
37
  this._jobsHistory = [];
38
+ this._nodesHistory = [];
37
39
  this._workersHistory = [];
38
40
  this._workUnitsHistory = [];
39
- this._histories = [this._jobsHistory, this._workersHistory, this._workUnitsHistory];
41
+ this._histories = [this._jobsHistory, this._nodesHistory, this._workersHistory, this._workUnitsHistory];
40
42
  this._queue = $('#jobs');
41
43
  this._workerInfo = $('#worker_info');
42
44
  this._disconnected = $('#disconnected');
43
45
  $(window).bind('resize', Console.renderGraphs);
44
- $('#workers .worker').live('click', Console.getWorkerInfo);
46
+ $('#nodes .worker').live('click', Console.getWorkerInfo);
47
+ $('#workers_legend').css({background : this.WORKERS_COLOR});
48
+ $('#nodes_legend').css({background : this.NODES_COLOR});
45
49
  this.getStatus();
46
50
  $.each(this.PRELOAD_IMAGES, function(){ var i = new Image(); i.src = this; });
47
51
  },
@@ -51,13 +55,14 @@ window.Console = {
51
55
  getStatus : function() {
52
56
  $.ajax({url : '/status', dataType : 'json', success : function(resp) {
53
57
  Console._jobs = resp.jobs;
54
- Console._workers = resp.workers;
58
+ Console._nodes = resp.nodes;
55
59
  Console._workUnitCount = resp.work_unit_count;
60
+ Console._workerCount = Console.countWorkers();
56
61
  Console.recordDataPoint();
57
62
  if (Console._disconnected.is(':visible')) Console._disconnected.fadeOut(Console.ANIMATION_SPEED);
58
63
  $('#queue').toggleClass('no_jobs', Console._jobs.length <= 0);
59
64
  Console.renderJobs();
60
- Console.renderWorkers();
65
+ Console.renderNodes();
61
66
  Console.renderGraphs();
62
67
  setTimeout(Console.getStatus, Console.POLL_INTERVAL);
63
68
  }, error : function(request, status, errorThrown) {
@@ -66,6 +71,13 @@ window.Console = {
66
71
  }});
67
72
  },
68
73
 
74
+ // Count the total number of workers in the current list of nodes.
75
+ countWorkers : function() {
76
+ var sum = 0;
77
+ for (var i=0; i < this._nodes.length; i++) sum += this._nodes[i].workers.length;
78
+ return sum;
79
+ },
80
+
69
81
  // Render an individual job afresh.
70
82
  renderJob : function(job) {
71
83
  this._queue.append('<div class="job" id="job_' + job.id + '" style="width:' + job.width + '%; background: #' + job.color + ';"><div class="completion ' + (job.percent_complete <= 0 ? 'zero' : '') + '" style="width:' + job.percent_complete + '%;"></div><div class="percent_complete">' + job.percent_complete + '%</div><div class="job_id">#' + job.id + '</div></div>');
@@ -105,12 +117,19 @@ window.Console = {
105
117
  },
106
118
 
107
119
  // Re-render all workers from scratch each time.
108
- renderWorkers : function() {
120
+ renderNodes : function() {
109
121
  var header = $('#sidebar_header');
110
- $('.has_workers', header).html(this._workers.length + " Active Worker Daemons");
111
- header.toggleClass('no_workers', this._workers.length <= 0);
112
- $('#workers').html($.map(this._workers, function(w) {
113
- return '<div class="worker ' + w.status + '" rel="' + w.name + '">' + w.name + '</div>';
122
+ var nc = this._nodes.length, wc = this._workerCount;
123
+ $('.has_nodes', header).html(nc + " Node" + (nc != 1 ? 's' : '') + " / " + wc + " Worker" + (wc != 1 ? 's' : ''));
124
+ header.toggleClass('no_nodes', this._nodes.length <= 0);
125
+ $('#nodes').html($.map(this._nodes, function(node) {
126
+ var html = "";
127
+ html += '<div class="node">' + node.host + '</div>';
128
+ html += $.map(node.workers, function(pid) {
129
+ var name = pid + '@' + node.host;
130
+ return '<div class="worker" rel="' + name + '">' + name + '</div>';
131
+ }).join('');
132
+ return html;
114
133
  }).join(''));
115
134
  },
116
135
 
@@ -118,7 +137,8 @@ window.Console = {
118
137
  recordDataPoint : function() {
119
138
  var timestamp = (new Date()).getTime();
120
139
  this._jobsHistory.push([timestamp, this._jobs.length]);
121
- this._workersHistory.push([timestamp, this._workers.length]);
140
+ this._nodesHistory.push([timestamp, this._nodes.length]);
141
+ this._workersHistory.push([timestamp, this._workerCount]);
122
142
  this._workUnitsHistory.push([timestamp, this._workUnitCount]);
123
143
  $.each(this._histories, function() {
124
144
  if (this.length > Console.MAX_DATA_POINTS) this.shift();
@@ -127,9 +147,16 @@ window.Console = {
127
147
 
128
148
  // Convert our recorded data points into a format Flot can understand.
129
149
  renderGraphs : function() {
130
- $.plot($('#jobs_graph'), [{label : 'Jobs in Queue', color : Console.JOBS_COLOR, data : Console._jobsHistory}], Console.GRAPH_OPTIONS);
131
- $.plot($('#workers_graph'), [{label : 'Active Workers', color : Console.WORKERS_COLOR, data : Console._workersHistory}], Console.GRAPH_OPTIONS);
132
- $.plot($('#work_units_graph'), [{label : 'Work Units in Queue', color : Console.WORK_UNITS_COLOR, data : Console._workUnitsHistory}], Console.GRAPH_OPTIONS);
150
+ $.plot($('#work_units_graph'), [
151
+ {label : 'Work Units in Queue', color : Console.WORK_UNITS_COLOR, data : Console._workUnitsHistory}
152
+ ], Console.GRAPH_OPTIONS);
153
+ $.plot($('#jobs_graph'), [
154
+ {label : 'Jobs in Queue', color : Console.JOBS_COLOR, data : Console._jobsHistory}
155
+ ], Console.GRAPH_OPTIONS);
156
+ $.plot($('#workers_graph'), [
157
+ {label : 'Nodes', color : Console.NODES_COLOR, data : Console._nodesHistory},
158
+ {label : 'Workers', color : Console.WORKERS_COLOR, data : Console._workersHistory}
159
+ ], Console.GRAPH_OPTIONS);
133
160
  },
134
161
 
135
162
  // Request the Worker info from the central server.
@@ -4,7 +4,7 @@ require 'test_helper'
4
4
  class FailingWorkUnitsTest < Test::Unit::TestCase
5
5
 
6
6
  should "retry work units when they fail" do
7
- browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::App))
7
+ browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
8
8
 
9
9
  browser.post '/jobs', :job => {
10
10
  'action' => 'failure_testing',
@@ -1,14 +1,14 @@
1
1
  require 'test_helper'
2
2
 
3
- class AppTest < Test::Unit::TestCase
3
+ class ServerTest < Test::Unit::TestCase
4
4
 
5
5
  include Rack::Test::Methods
6
6
 
7
7
  def app
8
- CloudCrowd::App
8
+ CloudCrowd::Server
9
9
  end
10
10
 
11
- context "The CloudCrowd::App (Sinatra)" do
11
+ context "The CloudCrowd::Server (Sinatra)" do
12
12
 
13
13
  setup do
14
14
  CloudCrowd::Job.destroy_all
@@ -17,7 +17,7 @@ class AppTest < Test::Unit::TestCase
17
17
 
18
18
  should "be able to render the Operations Center (GET /)" do
19
19
  get '/'
20
- assert last_response.body.include? '<div id="workers">'
20
+ assert last_response.body.include? '<div id="nodes">'
21
21
  assert last_response.body.include? '<div id="graphs">'
22
22
  end
23
23
 
@@ -29,17 +29,17 @@ class AppTest < Test::Unit::TestCase
29
29
  assert resp['work_unit_count'] == 2
30
30
  end
31
31
 
32
- should "be able to check in a worker daemon, and then check out a work unit" do
33
- put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
34
- assert last_response.successful? && last_response.empty?
35
- post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
36
- checked_out = JSON.parse(last_response.body)
37
- assert checked_out['action'] == 'graphics_magick'
38
- assert checked_out['attempts'] == 0
39
- assert checked_out['status'] == CloudCrowd::PROCESSING
40
- status_check = JSON.parse(get('/worker/101@localhost').body)
41
- assert checked_out == status_check
42
- end
32
+ # should "be able to check in a worker daemon, and then check out a work unit" do
33
+ # put '/worker', :name => '101@localhost', :thread_status => 'sleeping'
34
+ # assert last_response.successful? && last_response.empty?
35
+ # post '/work', :worker_name => '101@localhost', :worker_actions => 'graphics_magick'
36
+ # checked_out = JSON.parse(last_response.body)
37
+ # assert checked_out['action'] == 'graphics_magick'
38
+ # assert checked_out['attempts'] == 0
39
+ # assert checked_out['status'] == CloudCrowd::PROCESSING
40
+ # status_check = JSON.parse(get('/worker/101@localhost').body)
41
+ # assert checked_out == status_check
42
+ # end
43
43
 
44
44
  should "have a heartbeat" do
45
45
  assert get('/heartbeat').body == 'buh-bump'
@@ -6,7 +6,7 @@ class WordCountTest < Test::Unit::TestCase
6
6
 
7
7
  setup do
8
8
  @asset_store = AssetStore.new
9
- @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::App))
9
+ @browser = Rack::Test::Session.new(Rack::MockSession.new(CloudCrowd::Server))
10
10
  @browser.put('/worker', :name => 'test_worker', :thread_status => 'sleeping')
11
11
  post_job_to_count_words_in_this_file
12
12
  @job_id = JSON.parse(@browser.last_response.body)['id']
@@ -24,15 +24,9 @@ class WordCountTest < Test::Unit::TestCase
24
24
  end
25
25
 
26
26
  should "be able to perform the processing stage of a word_count" do
27
- @browser.post('/work', :worker_name => 'test_worker', :worker_actions => 'word_count')
28
- assert @browser.last_response.ok?
29
- info = JSON.parse(@browser.last_response.body)
30
- assert info['status'] == 1
31
- assert info['action'] == 'word_count'
32
- assert info['input'] == "file://#{File.expand_path(__FILE__)}"
33
- action = CloudCrowd.actions['word_count'].new(info['status'], info['input'], info['options'], @asset_store)
27
+ action = CloudCrowd.actions['word_count'].new(1, "file://#{File.expand_path(__FILE__)}", {}, @asset_store)
34
28
  count = action.process
35
- assert count == 128
29
+ assert count == 104
36
30
  end
37
31
 
38
32
  end
data/test/blueprints.rb CHANGED
@@ -11,7 +11,6 @@ end
11
11
  CloudCrowd::WorkUnit.blueprint do
12
12
  job { CloudCrowd::Job.make }
13
13
  status { CloudCrowd::PROCESSING }
14
- worker_record_id { nil }
15
14
  input { Sham.url }
16
15
  action { 'graphics_magick' }
17
16
  end
@@ -13,5 +13,5 @@ CloudCrowd.configure(File.dirname(__FILE__) + '/config.yml')
13
13
  CloudCrowd.configure_database(File.dirname(__FILE__) + '/database.yml')
14
14
 
15
15
  map '/' do
16
- run CloudCrowd::App
16
+ run CloudCrowd::Server
17
17
  end
@@ -1,6 +1,4 @@
1
- :num_workers: 4
2
- :min_worker_wait: 1
3
- :max_worker_wait: 20
1
+ :max_workers: 10
4
2
  :work_unit_retries: 3
5
3
 
6
4
  :central_server: http://localhost:9173
@@ -5,7 +5,7 @@ class ConfigurationTest < Test::Unit::TestCase
5
5
  context "CloudCrowd Configuration" do
6
6
 
7
7
  should "have read in config.yml" do
8
- assert CloudCrowd.config[:num_workers] == 4
8
+ assert CloudCrowd.config[:max_workers] == 10
9
9
  assert CloudCrowd.config[:storage] == 'filesystem'
10
10
  end
11
11
 
@@ -26,6 +26,7 @@ class JobTest < Test::Unit::TestCase
26
26
  should "know its completion status" do
27
27
  assert !@job.all_work_units_complete?
28
28
  @unit.update_attributes(:status => CloudCrowd::SUCCEEDED, :output => '{"output":"hello"}')
29
+ @job.check_for_completion
29
30
  assert @job.reload.all_work_units_complete?
30
31
  assert @job.percent_complete == 100
31
32
  assert @job.outputs == "[\"hello\"]"
@@ -21,8 +21,6 @@ class WorkUnitTest < Test::Unit::TestCase
21
21
  assert @unit.complete?
22
22
  @unit.status = CloudCrowd::FAILED
23
23
  assert @unit.complete?
24
- @unit.expects :check_for_job_completion
25
- @unit.save
26
24
  end
27
25
 
28
26
  should "have JSON that includes job attributes" do
@@ -34,10 +32,10 @@ class WorkUnitTest < Test::Unit::TestCase
34
32
  end
35
33
 
36
34
  should "be able to retry, on failure" do
37
- @unit.update_attribute :worker_record_id, 10
35
+ @unit.update_attribute :worker_pid, 7337
38
36
  assert @unit.attempts == 0
39
37
  @unit.fail('oops', 10)
40
- assert @unit.worker_record == nil
38
+ assert @unit.worker_pid == nil
41
39
  assert @unit.attempts == 1
42
40
  assert @unit.processing?
43
41
  @unit.fail('oops again', 10)
data/views/index.erb CHANGED
@@ -36,25 +36,30 @@
36
36
  <div id="sidebar_background"></div>
37
37
  <div id="sidebar_bottom" class="sidebar_back"></div>
38
38
  <div id="sidebar_header" class="small_caps">
39
- <span class="has_workers">Active Worker Daemons</span>
40
- <span class="no_workers">No Workers Running&hellip;</span>
39
+ <span class="has_nodes"></span>
40
+ <span class="no_nodes">No Nodes Online&hellip;</span>
41
41
  </div>
42
- <div id="workers">
43
- <%# Render target for workers. %>
42
+ <div id="nodes">
43
+ <%# Render target for nodes and workers. %>
44
44
  </div>
45
45
  </div>
46
46
 
47
47
  <div id="graphs">
48
48
  <div class="graph_container">
49
- <div class="graph_title">Work Units in Queue:</div>
49
+ <div class="graph_title">Work Units in Queue</div>
50
50
  <div id="work_units_graph" class="graph"></div>
51
51
  </div>
52
52
  <div class="graph_container">
53
- <div class="graph_title">Jobs in Queue:</div>
53
+ <div class="graph_title">Jobs in Queue</div>
54
54
  <div id="jobs_graph" class="graph"></div>
55
55
  </div>
56
56
  <div class="graph_container">
57
- <div class="graph_title">Active Worker Daemons:</div>
57
+ <div class="graph_title">
58
+ Active Nodes
59
+ (<span id="nodes_legend" class="legend_box"></span>)
60
+ and Workers
61
+ (<span id="workers_legend" class="legend_box"></span>)
62
+ </div>
58
63
  <div id="workers_graph" class="graph"></div>
59
64
  </div>
60
65
  </div>
@@ -67,7 +72,7 @@
67
72
  <div id="worker_job_id">job #<span class="job_id"></span> / work unit #<span class="work_unit_id"></span></div>
68
73
  </div>
69
74
  <div id="worker_sleeping">
70
- worker is sleeping&hellip;
75
+ worker exiting&hellip;
71
76
  </div>
72
77
  </div>
73
78
  </div>
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: documentcloud-cloud-crowd
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Ashkenas
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-09-01 00:00:00 -07:00
12
+ date: 2009-09-15 00:00:00 -07:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -63,14 +63,14 @@ dependencies:
63
63
  version: 1.10.0
64
64
  version:
65
65
  - !ruby/object:Gem::Dependency
66
- name: daemons
66
+ name: thin
67
67
  type: :runtime
68
68
  version_requirement:
69
69
  version_requirements: !ruby/object:Gem::Requirement
70
70
  requirements:
71
71
  - - ">="
72
72
  - !ruby/object:Gem::Version
73
- version: 1.0.10
73
+ version: 1.2.4
74
74
  version:
75
75
  - !ruby/object:Gem::Dependency
76
76
  name: faker
@@ -144,23 +144,22 @@ files:
144
144
  - examples/word_count_example.rb
145
145
  - lib/cloud-crowd.rb
146
146
  - lib/cloud_crowd/action.rb
147
- - lib/cloud_crowd/app.rb
148
147
  - lib/cloud_crowd/asset_store/filesystem_store.rb
149
148
  - lib/cloud_crowd/asset_store/s3_store.rb
150
149
  - lib/cloud_crowd/asset_store.rb
151
150
  - lib/cloud_crowd/command_line.rb
152
- - lib/cloud_crowd/daemon.rb
153
151
  - lib/cloud_crowd/exceptions.rb
154
152
  - lib/cloud_crowd/helpers/authorization.rb
155
153
  - lib/cloud_crowd/helpers/resources.rb
156
154
  - lib/cloud_crowd/helpers.rb
157
155
  - lib/cloud_crowd/inflector.rb
158
156
  - lib/cloud_crowd/models/job.rb
157
+ - lib/cloud_crowd/models/node_record.rb
159
158
  - lib/cloud_crowd/models/work_unit.rb
160
- - lib/cloud_crowd/models/worker_record.rb
161
159
  - lib/cloud_crowd/models.rb
162
- - lib/cloud_crowd/runner.rb
160
+ - lib/cloud_crowd/node.rb
163
161
  - lib/cloud_crowd/schema.rb
162
+ - lib/cloud_crowd/server.rb
164
163
  - lib/cloud_crowd/worker.rb
165
164
  - LICENSE
166
165
  - public/css/admin_console.css
@@ -171,6 +170,7 @@ files:
171
170
  - public/images/header_back.png
172
171
  - public/images/logo.png
173
172
  - public/images/queue_fill.png
173
+ - public/images/server.png
174
174
  - public/images/server_error.png
175
175
  - public/images/sidebar_bottom.png
176
176
  - public/images/sidebar_top.png
@@ -181,7 +181,7 @@ files:
181
181
  - public/js/flot.js
182
182
  - public/js/jquery.js
183
183
  - README
184
- - test/acceptance/test_app.rb
184
+ - test/acceptance/test_server.rb
185
185
  - test/acceptance/test_failing_work_units.rb
186
186
  - test/acceptance/test_word_count.rb
187
187
  - test/blueprints.rb
@@ -1,95 +0,0 @@
1
- CloudCrowd.configure(ENV['CLOUD_CROWD_CONFIG'])
2
-
3
- module CloudCrowd
4
-
5
- # A CloudCrowd::Daemon, started by the Daemons gem, runs a CloudCrowd::Worker in
6
- # a loop, continually fetching and processing WorkUnits from the central
7
- # server.
8
- #
9
- # The Daemon backs off and pings the central server less frequently
10
- # when there isn't any work to be done, and speeds back up when there is.
11
- #
12
- # The `crowd` command responds to all the usual methods that the Daemons gem
13
- # supports.
14
- class Daemon
15
-
16
- # The back-off factor used to slow down requests for new work units
17
- # when the queue is empty.
18
- WAIT_MULTIPLIER = 1.5
19
-
20
- MIN_WAIT = CloudCrowd.config[:min_worker_wait]
21
- MAX_WAIT = CloudCrowd.config[:max_worker_wait]
22
-
23
- def initialize
24
- @wait_time = MIN_WAIT
25
- @worker = Worker.new
26
- Signal.trap('INT') { kill_worker_and_exit }
27
- Signal.trap('KILL') { kill_worker_and_exit }
28
- Signal.trap('TERM') { kill_worker_and_exit }
29
- end
30
-
31
- # Spin up our worker and monitoring threads. The monitor's the boss, and
32
- # will feel no compunction in killing the worker thread if necessary.
33
- # Check in before starting up. If check in fails, there's no sense in going.
34
- def run
35
- @worker.check_in('starting')
36
- @work_thread = run_worker
37
- @monitor_thread = run_monitor
38
- @monitor_thread.join
39
- end
40
-
41
-
42
- private
43
-
44
- # Loop forever, fetching WorkUnits and processing them.
45
- def run_worker
46
- Thread.new do
47
- loop do
48
- @worker.fetch_work_unit
49
- if @worker.has_work?
50
- @wait_time = MIN_WAIT
51
- while @worker.has_work?
52
- @worker.run
53
- sleep 0.01 # So as to listen for incoming signals.
54
- end
55
- else
56
- @wait_time = [@wait_time * WAIT_MULTIPLIER, MAX_WAIT].min
57
- sleep @wait_time
58
- end
59
- end
60
- end
61
- end
62
-
63
- # Checks in to let the central server know it's still alive every
64
- # CHECK_IN_INTERVAL seconds. Restarts the work_thread if it has died.
65
- def run_monitor
66
- Thread.new do
67
- sleep Worker::CHECK_IN_INTERVAL
68
- loop do
69
- @work_thread = run_monitor unless @work_thread.alive? || @exit_started
70
- @worker.check_in(@work_thread.status)
71
- sleep Worker::CHECK_IN_INTERVAL
72
- end
73
- end
74
- end
75
-
76
- def running?
77
- @work_thread.alive? || @monitor_thread.alive?
78
- end
79
-
80
- # At exit, kill the worker thread, gently at first, then forcefully.
81
- def kill_worker_and_exit
82
- @worker.check_out
83
- @exit_started = Time.now
84
- @work_thread.kill && @monitor_thread.kill
85
- sleep 0.3 while running? && Time.now - @exit_started < WORKER_EXIT_WAIT
86
- return Process.exit unless running?
87
- @work_thread.kill! && @monitor_thread.kill!
88
- Process.exit
89
- end
90
-
91
- end
92
-
93
- end
94
-
95
- CloudCrowd::Daemon.new.run
@@ -1,61 +0,0 @@
1
- module CloudCrowd
2
-
3
- # A WorkerRecord is a recording of an active worker daemon running remotely.
4
- # Every time it checks in, we keep track of its status. The attributes shown
5
- # here may lag their actual values by up to Worker::CHECK_IN_INTERVAL seconds.
6
- class WorkerRecord < ActiveRecord::Base
7
-
8
- EXPIRES_AFTER = 2 * Worker::CHECK_IN_INTERVAL
9
-
10
- has_one :work_unit
11
-
12
- validates_presence_of :name, :thread_status
13
-
14
- before_destroy :clear_work_units
15
-
16
- named_scope :alive, lambda { {:conditions => ['updated_at > ?', Time.now - EXPIRES_AFTER]} }
17
- named_scope :dead, lambda { {:conditions => ['updated_at <= ?', Time.now - EXPIRES_AFTER]} }
18
-
19
- # Save a Worker's current status to the database.
20
- def self.check_in(params)
21
- attrs = {:thread_status => params[:thread_status], :updated_at => Time.now}
22
- self.find_or_create_by_name(params[:name]).update_attributes!(attrs)
23
- end
24
-
25
- # Remove a terminated Worker's record from the database.
26
- def self.check_out(params)
27
- self.find_by_name(params[:name]).destroy
28
- end
29
-
30
- # We consider the worker to be alive if it's checked in more recently
31
- # than twice the expected interval ago.
32
- def alive?
33
- updated_at > Time.now - EXPIRES_AFTER
34
- end
35
-
36
- # Derive the Worker's PID on the remote machine from the name.
37
- def pid
38
- @pid ||= self.name.split('@').first
39
- end
40
-
41
- # Derive the hostname from the Worker's name.
42
- def hostname
43
- @hostname ||= self.name.split('@').last
44
- end
45
-
46
- def to_json(opts={})
47
- {
48
- 'name' => name,
49
- 'status' => work_unit && work_unit.display_status,
50
- }.to_json
51
- end
52
-
53
-
54
- private
55
-
56
- def clear_work_units
57
- WorkUnit.update_all('worker_record_id = null', "worker_record_id = #{id}")
58
- end
59
-
60
- end
61
- end
@@ -1,15 +0,0 @@
1
- # This is the script that kicks off a single CloudCrowd::Daemon. Rely on
2
- # cloud-crowd.rb for autoloading of all the code we need.
3
-
4
- require "#{File.dirname(__FILE__)}/../cloud-crowd"
5
-
6
- FileUtils.mkdir('log') unless File.exists?('log')
7
-
8
- Daemons.run("#{CloudCrowd::ROOT}/lib/cloud_crowd/daemon.rb", {
9
- :app_name => "cloud_crowd_worker",
10
- :dir_mode => :normal,
11
- :dir => 'log',
12
- :multiple => true,
13
- :backtrace => true,
14
- :log_output => true
15
- })