right_chimp 1.0.3 → 1.0.7

Sign up to get free protection for your applications and to get access to all the features.
data/CHANGES CHANGED
@@ -101,3 +101,15 @@ Version 1.0.2
101
101
  Version 1.0.3
102
102
  -------------
103
103
  * Bugfix: fix race condition in chimpd serial groups
104
+
105
+ Version 1.0.4
106
+ -------------
107
+ * Feature: chimp --chimpd-wait-until-done now blocks until a group has completed > 1 job
108
+ * Feature: chimp --chimpd-wait-until-done now returns immediately during a dry run
109
+ * Feature: chimpd client now log errors better, retries on failure
110
+ * Feature: chimp output when using chimpd is better behaved
111
+
112
+ Version 1.0.5
113
+ -------------
114
+ * Feature: improved error handling and logging when submitting to chimpd
115
+
data/Gemfile.lock CHANGED
@@ -1,11 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- right_chimp (1.0.2)
4
+ right_chimp (1.0.7)
5
+ nokogiri (~> 1.5.9)
5
6
  progressbar (~> 0.11.0)
6
7
  rake (~> 0.9.2.2)
7
8
  rest-client (~> 1.6.7)
8
- rest_connection (~> 1.0.6)
9
+ rest_connection (~> 1.0.10)
9
10
 
10
11
  GEM
11
12
  remote: https://rubygems.org/
@@ -14,23 +15,23 @@ GEM
14
15
  i18n (= 0.6.1)
15
16
  multi_json (~> 1.0)
16
17
  diff-lcs (1.1.3)
17
- highline (1.6.18)
18
+ highline (1.6.19)
18
19
  i18n (0.6.1)
19
- json (1.7.7)
20
+ json (1.8.0)
20
21
  mime-types (1.23)
21
- multi_json (1.7.3)
22
+ multi_json (1.7.7)
22
23
  net-ssh (2.6.7)
23
- nokogiri (1.5.9)
24
+ nokogiri (1.5.10)
24
25
  progressbar (0.11.0)
25
26
  rake (0.9.2.2)
26
27
  rest-client (1.6.7)
27
28
  mime-types (>= 1.16)
28
- rest_connection (1.0.6)
29
+ rest_connection (1.0.10)
29
30
  activesupport
30
31
  highline
31
32
  json
32
33
  net-ssh
33
- nokogiri
34
+ nokogiri (< 1.6.0)
34
35
  rest-client
35
36
  rspec (2.6.0)
36
37
  rspec-core (~> 2.6.0)
data/chimp.gemspec CHANGED
@@ -17,11 +17,12 @@ Gem::Specification.new do |s|
17
17
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
18
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
19
  s.require_paths = ["lib"]
20
-
21
- s.add_dependency "progressbar", "~> 0.11.0"
20
+
21
+ s.add_dependency "rest_connection", "~> 1.0.10"
22
22
  s.add_dependency "rest-client", "~> 1.6.7"
23
23
  s.add_dependency "rake", "~> 0.9.2.2"
24
- s.add_dependency "rest_connection", "~> 1.0.6"
24
+ s.add_dependency "nokogiri", "~> 1.5.9"
25
+ s.add_dependency "progressbar", "~> 0.11.0"
25
26
 
26
- s.add_development_dependency "rspec", "~> 2.6.0"
27
+ s.add_development_dependency "rspec", "~> 2.6.0"
27
28
  end
@@ -892,35 +892,24 @@ module Chimp
892
892
  #
893
893
  def chimpd_wait_until_done
894
894
  local_queue = ChimpQueue.instance
895
+ $stdout.print "Waiting for chimpd jobs to complete for group #{@group}..."
895
896
 
896
897
  begin
897
- while true
898
+ while !@dry_run
898
899
  local_queue = ChimpQueue.instance
899
900
 
900
901
  #
901
902
  # load up remote chimpd jobs into the local queue
902
903
  # this makes all the standard queue control methods available to us
903
904
  #
904
- retry_count = 1
905
905
  while true
906
906
  local_queue.reset!
907
907
 
908
908
  begin
909
- puts "Waiting for chimpd jobs to complete for group #{@group}..."
910
909
  all = ChimpDaemonClient.retrieve_group_info(@chimpd_host, @chimpd_port, @group, :all)
911
910
  rescue RestClient::ResourceNotFound
912
- if retry_count > 0
913
- retry_count -= 1
914
- sleep 5
915
- retry
916
- end
917
-
918
- if @ignore_errors
919
- exit 0
920
- else
921
- $stderr.puts "ERROR: Group \"#{group}\" not found!"
922
- exit 1
923
- end
911
+ sleep 5
912
+ retry
924
913
  end
925
914
 
926
915
  ChimpQueue.instance.create_group(@group)
@@ -6,36 +6,38 @@
6
6
 
7
7
  module Chimp
8
8
  class ChimpDaemon
9
- attr_accessor :verbose, :debug, :port, :concurrency, :delay, :retry_count, :dry_run, :logfile
9
+ attr_accessor :verbose, :debug, :port, :concurrency, :delay, :retry_count, :dry_run, :logfile, :chimp_queue
10
10
  attr_reader :queue, :running
11
-
11
+
12
12
  include Singleton
13
-
13
+
14
14
  def initialize
15
15
  @verbose = false
16
16
  @debug = false
17
17
  @port = 9055
18
18
  @concurrency = 50
19
19
  @delay = 0
20
- @retry_count = 0
20
+ @retry_count = 0
21
21
  @threads = []
22
22
  @running = false
23
23
  @queue = ChimpQueue.instance
24
+ @chimp_queue = Queue.new
24
25
  end
25
-
26
+
26
27
  #
27
28
  # Main entry point for chimpd command line application
28
29
  #
29
30
  def run
30
31
  install_signal_handlers
31
32
  parse_command_line
32
-
33
+
33
34
  puts "chimpd #{VERSION} launching with #{@concurrency} workers"
34
35
  spawn_queue_runner
35
36
  spawn_webserver
37
+ spawn_chimpd_submission_processor
36
38
  run_forever
37
39
  end
38
-
40
+
39
41
  #
40
42
  # Parse chimpd command line options
41
43
  #
@@ -51,7 +53,7 @@ module Chimp
51
53
  [ '--port', '-p', GetoptLong::REQUIRED_ARGUMENT ],
52
54
  [ '--exit', '-x', GetoptLong::NO_ARGUMENT ]
53
55
  )
54
-
56
+
55
57
  opts.each do |opt, arg|
56
58
  case opt
57
59
  when '--logfile', '-l'
@@ -79,52 +81,58 @@ module Chimp
79
81
  puts "Syntax: chimpd [--logfile=<name>] [--concurrency=<c>] [--delay=<d>] [--retry=<r>] [--port=<p>] [--verbose]"
80
82
  exit 1
81
83
  end
82
-
84
+
83
85
  #
84
86
  # Set up logging/verbosity
85
87
  #
86
88
  Chimp.set_verbose(@verbose, @quiet)
87
-
89
+
88
90
  if not @verbose
89
91
  ENV['REST_CONNECTION_LOG'] = "/dev/null"
90
92
  ENV['RESTCLIENT_LOG'] = "/dev/null"
91
93
  end
94
+
95
+ if @quiet
96
+ Log.threshold = Logger::WARN
97
+ end
98
+
92
99
  end
93
-
100
+
94
101
  #
95
102
  # Spawn the ChimpQueue threads
96
103
  #
97
- def spawn_queue_runner
104
+ def spawn_queue_runner
98
105
  @queue.max_threads = @concurrency
99
106
  @queue.delay = @delay
100
107
  @queue.retry_count = @retry_count
101
108
  @queue.start
102
109
  @running = true
103
110
  end
104
-
111
+
105
112
  #
106
113
  # Spawn a WEBrick Web server
107
114
  #
108
115
  def spawn_webserver
109
116
  opts = {
110
- :BindAddress => "localhost",
117
+ :BindAddress => "localhost",
111
118
  :Port => @port,
112
- :MaxClients => 50,
113
- :RequestTimeout => 60
119
+ :MaxClients => 500,
120
+ :RequestTimeout => 120,
121
+ :DoNotReverseLookup => true
114
122
  }
115
-
123
+
116
124
  if not @verbose
117
125
  opts[:Logger] = WEBrick::Log.new("/dev/null")
118
126
  opts[:AccessLog] = [nil, nil]
119
127
  end
120
128
 
121
- @server = ::WEBrick::HTTPServer.new(opts)
129
+ @server = ::WEBrick::HTTPServer.new(opts)
122
130
  @server.mount('/', DisplayServlet)
123
131
  @server.mount('/display', DisplayServlet)
124
132
  @server.mount('/job', JobServlet)
125
133
  @server.mount('/group', GroupServlet)
126
134
  @server.mount('/admin', AdminServlet)
127
-
135
+
128
136
  #
129
137
  # WEBrick threads
130
138
  #
@@ -132,7 +140,7 @@ module Chimp
132
140
  @server.start
133
141
  end
134
142
  end
135
-
143
+
136
144
  #
137
145
  # Process requests forever until we're killed
138
146
  #
@@ -144,18 +152,18 @@ module Chimp
144
152
  end
145
153
  end
146
154
  end
147
-
155
+
148
156
  #
149
157
  # Trap signals to exit cleanly
150
158
  #
151
159
  def install_signal_handlers
152
- ['INT', 'TERM'].each do |signal|
160
+ ['INT', 'TERM'].each do |signal|
153
161
  trap(signal) do
154
162
  self.quit
155
163
  end
156
164
  end
157
165
  end
158
-
166
+
159
167
  #
160
168
  # Quit by waiting for all chimp jobs to finish, not allowing
161
169
  # new jobs on the queue, and killing the web server.
@@ -168,7 +176,51 @@ module Chimp
168
176
  sleep 5
169
177
  exit 0
170
178
  end
171
-
179
+
180
+ #
181
+ # Spawn threads to process submitted requests
182
+ #
183
+ def spawn_chimpd_submission_processor
184
+ n = @concurrency/4
185
+ n = 10 if n < 10
186
+ Log.debug "Logging into API..."
187
+
188
+ #
189
+ # There is a race condition logging in with rest_connection.
190
+ # As a workaround, do a tag query first thing when chimpd starts.
191
+ #
192
+ begin
193
+ c = Chimp.new
194
+ c.interactive = false
195
+ c.quiet = true
196
+ c.tags = ["bogus:tag=true"]
197
+ c.run
198
+ rescue StandardError
199
+ end
200
+
201
+ Log.debug "Spawning #{n} submission processing threads"
202
+ (1..n).each do |n|
203
+ @threads ||=[]
204
+ @threads << Thread.new {
205
+ while true
206
+ begin
207
+ queued_request = @chimp_queue.pop
208
+ group = queued_request.group
209
+ queued_request.interactive = false
210
+
211
+ tasks = queued_request.process
212
+ tasks.each do |task|
213
+ ChimpQueue.instance.push(group, task)
214
+ end
215
+
216
+ rescue StandardError => ex
217
+ Log.error "submission processor: group=\"#{group}\" script=\"#{queued_request.script}\": #{ex}"
218
+ end
219
+ end
220
+ }
221
+ end
222
+ end
223
+
172
224
  #
173
225
  # GenericServlet -- servlet superclass
174
226
  #
@@ -176,13 +228,13 @@ module Chimp
176
228
  def get_verb(req)
177
229
  r = req.request_uri.path.split('/')[2]
178
230
  end
179
-
231
+
180
232
  def get_id(req)
181
233
  uri_parts = req.request_uri.path.split('/')
182
234
  id = uri_parts[-2]
183
235
  return id
184
236
  end
185
-
237
+
186
238
  #
187
239
  # Get the body of the request-- assume YAML
188
240
  #
@@ -194,7 +246,7 @@ module Chimp
194
246
  end
195
247
  end
196
248
  end # GenericServlet
197
-
249
+
198
250
  #
199
251
  # AdminServlet - admin functions
200
252
  #
@@ -206,11 +258,11 @@ module Chimp
206
258
  if shutdown == true
207
259
  ChimpDaemon.instance.quit
208
260
  end
209
-
210
- raise WEBrick::HTTPStatus::OK
261
+
262
+ raise WEBrick::HTTPStatus::OK
211
263
  end
212
- end # AdminServlet
213
-
264
+ end # AdminServlet
265
+
214
266
  #
215
267
  # GroupServlet - group information and control
216
268
  #
@@ -223,17 +275,17 @@ module Chimp
223
275
  #
224
276
  def do_GET(req, resp)
225
277
  jobs = {}
226
-
278
+
227
279
  group_name = req.request_uri.path.split('/')[-2]
228
280
  filter = req.request_uri.path.split('/')[-1]
229
-
281
+
230
282
  g = ChimpQueue[group_name.to_sym]
231
283
  raise WEBrick::HTTPStatus::NotFound, "Group not found" unless g
232
284
  jobs = g.get_jobs_by_status(filter)
233
285
  resp.body = jobs.to_yaml
234
286
  raise WEBrick::HTTPStatus::OK
235
287
  end
236
-
288
+
237
289
  #
238
290
  # POST to a group to trigger a group action
239
291
  # /group/<name>/<action>
@@ -245,21 +297,21 @@ module Chimp
245
297
 
246
298
  if filter == 'create'
247
299
  ChimpQueue.instance.create_group(group_name, payload['type'], payload['concurrency'])
248
-
300
+
249
301
  elsif filter == 'retry'
250
302
  group = ChimpQueue[group_name.to_sym]
251
303
  raise WEBrick::HTTPStatus::NotFound, "Group not found" unless group
252
-
304
+
253
305
  group.requeue_failed_jobs!
254
306
  raise WEBrick::HTTPStatus::OK
255
-
307
+
256
308
  else
257
309
  raise WEBrick::HTTPStatus::PreconditionFailed.new("invalid action")
258
310
  end
259
311
  end
260
-
312
+
261
313
  end
262
-
314
+
263
315
  #
264
316
  # JobServlet - job control
265
317
  #
@@ -273,20 +325,23 @@ module Chimp
273
325
 
274
326
  payload = self.get_payload(req)
275
327
  raise WEBrick::HTTPStatus::PreconditionFailed.new('missing payload') unless payload
276
-
328
+
277
329
  q = ChimpQueue.instance
278
330
  group = payload.group
279
-
331
+
280
332
  #
281
333
  # Ask chimpd to process a Chimp object directly
282
334
  #
335
+ #if verb == 'process' or verb == 'add'
336
+ # payload.interactive = false
337
+ # tasks = payload.process
338
+ # tasks.each do |task|
339
+ # q.push(group, task)
340
+ # end
341
+
283
342
  if verb == 'process' or verb == 'add'
284
- payload.interactive = false
285
- tasks = payload.process
286
- tasks.each do |task|
287
- q.push(group, task)
288
- end
289
-
343
+ ChimpDaemon.instance.chimp_queue.push payload
344
+ id = 0
290
345
  elsif verb == 'update'
291
346
  puts "UPDATE"
292
347
  q.get_job(job_id).status = payload.status
@@ -295,28 +350,28 @@ module Chimp
295
350
  resp.body = {
296
351
  'id' => id
297
352
  }.to_yaml
298
-
353
+
299
354
  raise WEBrick::HTTPStatus::OK
300
355
  end
301
-
356
+
302
357
  def do_GET(req, resp)
303
358
  id = self.get_id(req)
304
359
  verb = self.get_verb(req)
305
360
  job_results = "OK"
306
361
  queue = ChimpQueue.instance
307
-
362
+
308
363
  #
309
364
  # check for special job ids
310
365
  #
311
- jobs = []
366
+ jobs = []
312
367
  jobs << queue.get_job(id.to_i)
313
-
368
+
314
369
  jobs = queue.get_jobs_by_status(:running) if id == 'running'
315
370
  jobs = queue.get_jobs_by_status(:error) if id == 'error'
316
371
  jobs = queue.get_jobs if id == 'all'
317
-
372
+
318
373
  raise WEBrick::HTTPStatus::PreconditionFailed.new('invalid or missing job_id #{id}') unless jobs.size > 0
319
-
374
+
320
375
  #
321
376
  # ACK a job -- mark it as successful even if it failed
322
377
  #
@@ -324,9 +379,9 @@ module Chimp
324
379
  jobs.each do |j|
325
380
  j.status = Executor::STATUS_DONE
326
381
  end
327
-
382
+
328
383
  resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
329
-
384
+
330
385
  #
331
386
  # retry a job
332
387
  #
@@ -334,9 +389,9 @@ module Chimp
334
389
  jobs.each do |j|
335
390
  j.requeue
336
391
  end
337
-
392
+
338
393
  resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
339
-
394
+
340
395
  #
341
396
  # cancel an active job
342
397
  #
@@ -344,9 +399,9 @@ module Chimp
344
399
  jobs.each do |j|
345
400
  j.cancel if j.respond_to? :cancel
346
401
  end
347
-
402
+
348
403
  resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
349
-
404
+
350
405
  #
351
406
  # produce a report
352
407
  #
@@ -355,17 +410,17 @@ module Chimp
355
410
  jobs.each do |j|
356
411
  results << [j.group.group_id, j.class.to_s.sub("Chimp::",""), j.job_id, j.info, j.target, j.time_start, j.time_end, j.get_total_exec_time, j.status].join(",")
357
412
  end
358
-
413
+
359
414
  queue.group.values.each do |g|
360
415
  results << [g.group_id, g.class.to_s.sub("Chimp::",""), "", "", "", g.time_start, g.time_end, g.get_total_exec_time, ""].join(",")
361
416
  end
362
-
417
+
363
418
  job_results = results.join("\n") + "\n"
364
-
419
+
365
420
  resp['Content-type'] = "text/csv"
366
421
  resp['Content-disposition'] = "attachment;filename=chimp.csv"
367
422
  end
368
-
423
+
369
424
  #
370
425
  # return a list of the results
371
426
  #
@@ -373,7 +428,7 @@ module Chimp
373
428
  raise WEBrick::HTTPStatus::OK
374
429
  end
375
430
  end # JobServlet
376
-
431
+
377
432
  #
378
433
  # DisplayServlet
379
434
  #
@@ -387,14 +442,14 @@ module Chimp
387
442
  else
388
443
  template_file_name = 'lib/right_chimp/templates/all_jobs.erb'
389
444
  end
390
-
445
+
391
446
  @template = ERB.new(File.read(template_file_name), nil, ">")
392
447
  end
393
-
394
- queue = ChimpQueue.instance
448
+
449
+ queue = ChimpQueue.instance
395
450
  jobs = queue.get_jobs
396
451
  group_name = nil
397
-
452
+
398
453
  if job_filter == "group"
399
454
  group_name = req.request_uri.path.split('/')[-1]
400
455
  g = ChimpQueue[group_name.to_sym]
@@ -405,12 +460,10 @@ module Chimp
405
460
  count_jobs_queued = queue.get_jobs_by_status(:none).size
406
461
  count_jobs_failed = queue.get_jobs_by_status(:error).size
407
462
  count_jobs_done = queue.get_jobs_by_status(:done).size
408
-
463
+
409
464
  resp.body = @template.result(binding)
410
465
  raise WEBrick::HTTPStatus::OK
411
466
  end
412
467
  end # DisplayServlet
413
- end
468
+ end # ChimpDaemon
414
469
  end
415
-
416
-
@@ -1,6 +1,6 @@
1
1
  module Chimp
2
2
  #
3
- # The ChimpDaemonClient contains the code for communicating with the
3
+ # The ChimpDaemonClient contains the code for communicating with the
4
4
  # RESTful chimpd Web service
5
5
  #
6
6
  class ChimpDaemonClient
@@ -9,23 +9,53 @@ module Chimp
9
9
  #
10
10
  def self.submit(host, port, chimp_object)
11
11
  uri = "http://#{host}:#{port}/job/process"
12
- response = RestClient.post uri, chimp_object.to_yaml
13
-
14
- if response.code > 199 and response.code < 300
15
- begin
12
+ attempts = 3
13
+
14
+ begin
15
+ response = RestClient.post uri, chimp_object.to_yaml
16
+
17
+ if response.code > 199 and response.code < 300
16
18
  id = YAML::load(response.body)['id']
17
- rescue StandardError => ex
18
- puts ex
19
+ return true
20
+ else
21
+ $stderr.puts "WARNING: error submitting to chimpd! response code: #{reponse.code}"
22
+ return false
19
23
  end
20
- else
21
- $stderr.puts "error submitting to chimpd! response code: #{reponse.code}"
24
+
25
+ rescue RestClient::RequestTimeout => ex
26
+ $stderr.puts "WARNING: Request timeout talking to chimpd for job #{chimp_object.script}: #{ex.message} (#{ex.http_code})"
27
+ attempts -= 1
28
+ sleep 5 and retry if attempts > 0
29
+ return false
30
+
31
+ rescue RestClient::InternalServerError => ex
32
+ $stderr.puts "WARNING: Error submitting job to chimpd: #{error_message}, retrying..."
33
+ attempts -= 1
34
+ sleep 5 and retry if attempts > 0
35
+ return false
36
+
37
+ rescue Errno::ECONNRESET => ex
38
+ $stderr.puts "WARNING: Connection reset by peer, retrying..."
39
+ attempts -= 1
40
+ sleep 5 and retry if attempts > 0
41
+ return false
42
+
43
+ rescue Errno::EPIPE => ex
44
+ $stderr.puts "WARNING: broken pipe, retrying..."
45
+ attempts -= 1
46
+ sleep 5 and retry if attempts > 0
47
+ return false
48
+
49
+ rescue Errno::ECONNREFUSED => ex
50
+ $stderr.puts "ERROR: connection refused, aborting"
51
+ return false
52
+
53
+ rescue RestClient::Exception => ex
54
+ $stderr.puts "ERROR: Error submitting job to chimpd #{chimp_object.script}: #{ex.message}"
22
55
  return false
23
56
  end
24
-
25
- puts "chimpd submission complete"
26
- return true
27
57
  end
28
-
58
+
29
59
  #
30
60
  # quit a remote chimpd
31
61
  #
@@ -43,32 +73,32 @@ module Chimp
43
73
  jobs = YAML::load(response.body)
44
74
  return jobs
45
75
  end
46
-
76
+
47
77
  def self.retrieve_group_info(host, port, group_name, status)
48
78
  uri = "http://#{host}:#{port}/group/#{group_name}/#{status}"
49
79
  response = RestClient.get uri
50
80
  group = YAML::load(response.body)
51
81
  return group
52
82
  end
53
-
83
+
54
84
  def self.set_job_status(host, port, job_id, status)
55
85
  uri = "http://#{host}:#{port}/job/#{job_id}/update"
56
86
  response = RestClient.post uri, { :status => status}
57
87
  return YAML::load(response.body)
58
88
  end
59
-
89
+
60
90
  def self.create_group(host, port, name, type, concurrency)
61
91
  uri = "http://#{host}:#{port}/group/#{name}/create"
62
92
  payload = { 'type' => type, 'concurrency' => concurrency }.to_yaml
63
93
  response = RestClient.post(uri, payload)
64
94
  return YAML::load(response.body)
65
95
  end
66
-
96
+
67
97
  def self.retry_group(host, port, group_name)
68
98
  uri = "http://#{host}:#{port}/group/#{group_name}/retry"
69
99
  response = RestClient.post(uri, {})
70
100
  return YAML::load(response.body)
71
101
  end
72
-
102
+
73
103
  end
74
- end
104
+ end
@@ -1,5 +1,5 @@
1
1
  module Chimp
2
-
2
+
3
3
  #
4
4
  # Factory
5
5
  #
@@ -14,7 +14,7 @@ module Chimp
14
14
  end
15
15
  end
16
16
  end
17
-
17
+
18
18
  #
19
19
  # An ExecutionGroup contains a set of Executors to be processed
20
20
  #
@@ -24,7 +24,7 @@ module Chimp
24
24
  class ExecutionGroup
25
25
  attr_accessor :group_id, :description, :concurrency
26
26
  attr_reader :time_start, :time_end
27
-
27
+
28
28
  def initialize(new_group_id=nil)
29
29
  @group_id = new_group_id
30
30
  @queue = []
@@ -34,18 +34,18 @@ module Chimp
34
34
  @time_end = nil
35
35
  @concurrency = 1
36
36
  end
37
-
37
+
38
38
  #
39
39
  # Add something to the work queue
40
- #
40
+ #
41
41
  def push(j)
42
- raise "invalid work" if j == nil
42
+ raise "invalid work" if j == nil
43
43
  j.job_id = IDManager.get if j.job_id == nil
44
44
  j.group = self
45
45
  @queue.push(j)
46
46
  @jobs_by_id[j.job_id] = j
47
47
  end
48
-
48
+
49
49
  #
50
50
  # Take something from the queue
51
51
  #
@@ -54,7 +54,7 @@ module Chimp
54
54
  @time_start = Time.now if @time_start == nil
55
55
  return x
56
56
  end
57
-
57
+
58
58
  #
59
59
  # Return a hash of the results
60
60
  #
@@ -62,7 +62,7 @@ module Chimp
62
62
  return self.get_jobs.map do |task|
63
63
  next if task == nil
64
64
  next if task.server == nil
65
-
65
+
66
66
  {
67
67
  :job_id => task.job_id,
68
68
  :name => task.info,
@@ -76,14 +76,14 @@ module Chimp
76
76
  }
77
77
  end
78
78
  end
79
-
79
+
80
80
  #
81
81
  # Size of the active queue
82
82
  #
83
83
  def size
84
84
  return @queue.size
85
85
  end
86
-
86
+
87
87
  #
88
88
  # Sort queue by server nickname
89
89
  #
@@ -94,35 +94,35 @@ module Chimp
94
94
  end
95
95
  end
96
96
  end
97
-
97
+
98
98
  #
99
99
  # Reset the queue
100
100
  #
101
101
  def reset!
102
102
  @queue = []
103
103
  end
104
-
104
+
105
105
  #
106
106
  # Get all jobs
107
107
  #
108
108
  def get_jobs
109
109
  @jobs_by_id.values
110
110
  end
111
-
111
+
112
112
  #
113
113
  # Get all job ids
114
114
  #
115
115
  def get_job_ids
116
116
  @jobs_by_id.keys
117
117
  end
118
-
118
+
119
119
  #
120
120
  # Get a particular job
121
121
  #
122
122
  def get_job(i)
123
123
  @jobs_by_id[i]
124
124
  end
125
-
125
+
126
126
  #
127
127
  # Get jobs by status
128
128
  #
@@ -133,11 +133,11 @@ module Chimp
133
133
  end
134
134
  return r
135
135
  end
136
-
136
+
137
137
  def job_completed
138
138
  @time_end = Time.now
139
139
  end
140
-
140
+
141
141
  #
142
142
  # Reset all jobs and bulk set them
143
143
  #
@@ -147,7 +147,7 @@ module Chimp
147
147
  self.push(job)
148
148
  end
149
149
  end
150
-
150
+
151
151
  #
152
152
  # An execution group is "ready" if it has work that can be done;
153
153
  # see implementation in child classes.
@@ -155,19 +155,24 @@ module Chimp
155
155
  def ready?
156
156
  raise "unimplemented"
157
157
  end
158
-
158
+
159
159
  #
160
160
  # An execution group is "done" if nothing is queued or running
161
+ # and at least one job has completed.
161
162
  #
162
163
  def done?
163
- return (get_jobs_by_status(Executor::STATUS_NONE).size == 0 && get_jobs_by_status(Executor::STATUS_RUNNING).size == 0)
164
+ return (
165
+ get_jobs_by_status(Executor::STATUS_NONE).size == 0 &&
166
+ get_jobs_by_status(Executor::STATUS_RUNNING).size == 0 &&
167
+ get_jobs_by_status(Executor::STATUS_DONE).size > 0
168
+ )
164
169
  end
165
-
170
+
166
171
  #
167
172
  # Is this execution group running anything?
168
173
  #
169
174
  def running?
170
- total_jobs_running = get_jobs_by_status(Executor::STATUS_NONE).size +
175
+ total_jobs_running = get_jobs_by_status(Executor::STATUS_NONE).size +
171
176
  get_jobs_by_status(Executor::STATUS_RUNNING).size +
172
177
  get_jobs_by_status(Executor::STATUS_RETRYING).size
173
178
  return(total_jobs_running > 0)
@@ -193,7 +198,7 @@ module Chimp
193
198
  job.time_end = nil
194
199
  self.push(job)
195
200
  end
196
-
201
+
197
202
  #
198
203
  # Cancel a job by id
199
204
  #
@@ -205,7 +210,7 @@ module Chimp
205
210
  job.time_end = Time.now
206
211
  @queue.delete(job)
207
212
  end
208
-
213
+
209
214
  #
210
215
  # Return total execution time
211
216
  #
@@ -218,18 +223,18 @@ module Chimp
218
223
  return @time_end.to_i- @time_start.to_i
219
224
  end
220
225
  end
221
-
226
+
222
227
  #
223
228
  # Print out ExecutionGroup information
224
229
  #
225
230
  def to_s
226
231
  return "#{self.class}[#{group_id}]: ready=#{self.ready?} total_jobs=#{@jobs_by_id.size} queued_jobs=#{self.size}"
227
232
  end
228
-
233
+
229
234
  ###################################
230
235
  protected
231
236
  ###################################
232
-
237
+
233
238
  #
234
239
  # Return total execution time or -1 for errors
235
240
  #
@@ -238,7 +243,7 @@ module Chimp
238
243
  end
239
244
 
240
245
  end
241
-
246
+
242
247
  #
243
248
  # SerialExecutionGroup: run only one job at a time
244
249
  #
@@ -246,12 +251,12 @@ module Chimp
246
251
  def ready?
247
252
  return get_jobs_by_status(Executor::STATUS_RUNNING).size == 0 && get_jobs_by_status(Executor::STATUS_NONE).size > 0
248
253
  end
249
-
254
+
250
255
  def short_name
251
256
  "S"
252
257
  end
253
258
  end
254
-
259
+
255
260
  #
256
261
  # ParallelExecutionGroup: run multiple jobs at once
257
262
  #
@@ -260,7 +265,7 @@ module Chimp
260
265
  super(new_group_id)
261
266
  @concurrency = 25
262
267
  end
263
-
268
+
264
269
  #
265
270
  # FIXME - we're not currently using the @concurrency setting to limit execution
266
271
  # due to an unknown bug...
@@ -268,7 +273,7 @@ module Chimp
268
273
  def ready?
269
274
  return (get_jobs_by_status(Executor::STATUS_NONE).size > 0) # and get_jobs_by_status(Executor::STATUS_RUNNING).size < @concurrency)
270
275
  end
271
-
276
+
272
277
  def short_name
273
278
  "P"
274
279
  end
@@ -1,3 +1,3 @@
1
1
  module Chimp
2
- VERSION = "1.0.3"
2
+ VERSION = "1.0.7"
3
3
  end
metadata CHANGED
@@ -1,32 +1,36 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: right_chimp
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.7
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Christopher Deutsch
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2013-06-05 00:00:00.000000000 Z
12
+ date: 2013-06-24 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
- name: progressbar
15
+ name: rest_connection
15
16
  requirement: !ruby/object:Gem::Requirement
17
+ none: false
16
18
  requirements:
17
19
  - - ~>
18
20
  - !ruby/object:Gem::Version
19
- version: 0.11.0
21
+ version: 1.0.10
20
22
  type: :runtime
21
23
  prerelease: false
22
24
  version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
23
26
  requirements:
24
27
  - - ~>
25
28
  - !ruby/object:Gem::Version
26
- version: 0.11.0
29
+ version: 1.0.10
27
30
  - !ruby/object:Gem::Dependency
28
31
  name: rest-client
29
32
  requirement: !ruby/object:Gem::Requirement
33
+ none: false
30
34
  requirements:
31
35
  - - ~>
32
36
  - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
34
38
  type: :runtime
35
39
  prerelease: false
36
40
  version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
37
42
  requirements:
38
43
  - - ~>
39
44
  - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
41
46
  - !ruby/object:Gem::Dependency
42
47
  name: rake
43
48
  requirement: !ruby/object:Gem::Requirement
49
+ none: false
44
50
  requirements:
45
51
  - - ~>
46
52
  - !ruby/object:Gem::Version
@@ -48,27 +54,47 @@ dependencies:
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
51
58
  requirements:
52
59
  - - ~>
53
60
  - !ruby/object:Gem::Version
54
61
  version: 0.9.2.2
55
62
  - !ruby/object:Gem::Dependency
56
- name: rest_connection
63
+ name: nokogiri
57
64
  requirement: !ruby/object:Gem::Requirement
65
+ none: false
58
66
  requirements:
59
67
  - - ~>
60
68
  - !ruby/object:Gem::Version
61
- version: 1.0.6
69
+ version: 1.5.9
62
70
  type: :runtime
63
71
  prerelease: false
64
72
  version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
65
74
  requirements:
66
75
  - - ~>
67
76
  - !ruby/object:Gem::Version
68
- version: 1.0.6
77
+ version: 1.5.9
78
+ - !ruby/object:Gem::Dependency
79
+ name: progressbar
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ~>
84
+ - !ruby/object:Gem::Version
85
+ version: 0.11.0
86
+ type: :runtime
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ~>
92
+ - !ruby/object:Gem::Version
93
+ version: 0.11.0
69
94
  - !ruby/object:Gem::Dependency
70
95
  name: rspec
71
96
  requirement: !ruby/object:Gem::Requirement
97
+ none: false
72
98
  requirements:
73
99
  - - ~>
74
100
  - !ruby/object:Gem::Version
@@ -76,6 +102,7 @@ dependencies:
76
102
  type: :development
77
103
  prerelease: false
78
104
  version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
79
106
  requirements:
80
107
  - - ~>
81
108
  - !ruby/object:Gem::Version
@@ -127,26 +154,27 @@ files:
127
154
  - spec/spec_selection.rb
128
155
  homepage: https://github.com/rightscale/right_chimp
129
156
  licenses: []
130
- metadata: {}
131
157
  post_install_message:
132
158
  rdoc_options: []
133
159
  require_paths:
134
160
  - lib
135
161
  required_ruby_version: !ruby/object:Gem::Requirement
162
+ none: false
136
163
  requirements:
137
- - - '>='
164
+ - - ! '>='
138
165
  - !ruby/object:Gem::Version
139
166
  version: '0'
140
167
  required_rubygems_version: !ruby/object:Gem::Requirement
168
+ none: false
141
169
  requirements:
142
- - - '>='
170
+ - - ! '>='
143
171
  - !ruby/object:Gem::Version
144
172
  version: '0'
145
173
  requirements: []
146
174
  rubyforge_project:
147
- rubygems_version: 2.0.3
175
+ rubygems_version: 1.8.23
148
176
  signing_key:
149
- specification_version: 4
177
+ specification_version: 3
150
178
  summary: RightScale platform command-line tool
151
179
  test_files:
152
180
  - spec/spec_chimp.rb
checksums.yaml DELETED
@@ -1,7 +0,0 @@
1
- ---
2
- SHA1:
3
- metadata.gz: 1562feef06b0bcc0b1c87bd55c8421abafd7ca26
4
- data.tar.gz: 724add71f798b34e6a4a4e37e2611c90d60e44b0
5
- SHA512:
6
- metadata.gz: 5aee4d9a399b1a0b7dc99b0b05640bb372ced0015a54f61caff3f9e7b0ab01186be690d94ffa5a0c1eaeb4fa2e8f5f07c68ddb547561630cfe4a5d15953c21bb
7
- data.tar.gz: 6d1ec02570a46c928512df6b0387759faad9a15b7774d4ed408a5fd14f693da8d1d5af97cbe8fc5d35eb5d052062d689ea60963f2d664979a4fdea3b50f3da58