right_chimp 1.0.3 → 1.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGES CHANGED
@@ -101,3 +101,15 @@ Version 1.0.2
101
101
  Version 1.0.3
102
102
  -------------
103
103
  * Bugfix: fix race condition in chimpd serial groups
104
+
105
+ Version 1.0.4
106
+ -------------
107
+ * Feature: chimp --chimpd-wait-until-done now blocks until a group has completed > 1 job
108
+ * Feature: chimp --chimpd-wait-until-done now returns immediately during a dry run
109
+ * Feature: chimpd client now log errors better, retries on failure
110
+ * Feature: chimp output when using chimpd is better behaved
111
+
112
+ Version 1.0.5
113
+ -------------
114
+ * Feature: improved error handling and logging when submitting to chimpd
115
+
data/Gemfile.lock CHANGED
@@ -1,11 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- right_chimp (1.0.2)
4
+ right_chimp (1.0.7)
5
+ nokogiri (~> 1.5.9)
5
6
  progressbar (~> 0.11.0)
6
7
  rake (~> 0.9.2.2)
7
8
  rest-client (~> 1.6.7)
8
- rest_connection (~> 1.0.6)
9
+ rest_connection (~> 1.0.10)
9
10
 
10
11
  GEM
11
12
  remote: https://rubygems.org/
@@ -14,23 +15,23 @@ GEM
14
15
  i18n (= 0.6.1)
15
16
  multi_json (~> 1.0)
16
17
  diff-lcs (1.1.3)
17
- highline (1.6.18)
18
+ highline (1.6.19)
18
19
  i18n (0.6.1)
19
- json (1.7.7)
20
+ json (1.8.0)
20
21
  mime-types (1.23)
21
- multi_json (1.7.3)
22
+ multi_json (1.7.7)
22
23
  net-ssh (2.6.7)
23
- nokogiri (1.5.9)
24
+ nokogiri (1.5.10)
24
25
  progressbar (0.11.0)
25
26
  rake (0.9.2.2)
26
27
  rest-client (1.6.7)
27
28
  mime-types (>= 1.16)
28
- rest_connection (1.0.6)
29
+ rest_connection (1.0.10)
29
30
  activesupport
30
31
  highline
31
32
  json
32
33
  net-ssh
33
- nokogiri
34
+ nokogiri (< 1.6.0)
34
35
  rest-client
35
36
  rspec (2.6.0)
36
37
  rspec-core (~> 2.6.0)
data/chimp.gemspec CHANGED
@@ -17,11 +17,12 @@ Gem::Specification.new do |s|
17
17
  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
18
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
19
  s.require_paths = ["lib"]
20
-
21
- s.add_dependency "progressbar", "~> 0.11.0"
20
+
21
+ s.add_dependency "rest_connection", "~> 1.0.10"
22
22
  s.add_dependency "rest-client", "~> 1.6.7"
23
23
  s.add_dependency "rake", "~> 0.9.2.2"
24
- s.add_dependency "rest_connection", "~> 1.0.6"
24
+ s.add_dependency "nokogiri", "~> 1.5.9"
25
+ s.add_dependency "progressbar", "~> 0.11.0"
25
26
 
26
- s.add_development_dependency "rspec", "~> 2.6.0"
27
+ s.add_development_dependency "rspec", "~> 2.6.0"
27
28
  end
@@ -892,35 +892,24 @@ module Chimp
892
892
  #
893
893
  def chimpd_wait_until_done
894
894
  local_queue = ChimpQueue.instance
895
+ $stdout.print "Waiting for chimpd jobs to complete for group #{@group}..."
895
896
 
896
897
  begin
897
- while true
898
+ while !@dry_run
898
899
  local_queue = ChimpQueue.instance
899
900
 
900
901
  #
901
902
  # load up remote chimpd jobs into the local queue
902
903
  # this makes all the standard queue control methods available to us
903
904
  #
904
- retry_count = 1
905
905
  while true
906
906
  local_queue.reset!
907
907
 
908
908
  begin
909
- puts "Waiting for chimpd jobs to complete for group #{@group}..."
910
909
  all = ChimpDaemonClient.retrieve_group_info(@chimpd_host, @chimpd_port, @group, :all)
911
910
  rescue RestClient::ResourceNotFound
912
- if retry_count > 0
913
- retry_count -= 1
914
- sleep 5
915
- retry
916
- end
917
-
918
- if @ignore_errors
919
- exit 0
920
- else
921
- $stderr.puts "ERROR: Group \"#{group}\" not found!"
922
- exit 1
923
- end
911
+ sleep 5
912
+ retry
924
913
  end
925
914
 
926
915
  ChimpQueue.instance.create_group(@group)
@@ -6,36 +6,38 @@
6
6
 
7
7
  module Chimp
8
8
  class ChimpDaemon
9
- attr_accessor :verbose, :debug, :port, :concurrency, :delay, :retry_count, :dry_run, :logfile
9
+ attr_accessor :verbose, :debug, :port, :concurrency, :delay, :retry_count, :dry_run, :logfile, :chimp_queue
10
10
  attr_reader :queue, :running
11
-
11
+
12
12
  include Singleton
13
-
13
+
14
14
  def initialize
15
15
  @verbose = false
16
16
  @debug = false
17
17
  @port = 9055
18
18
  @concurrency = 50
19
19
  @delay = 0
20
- @retry_count = 0
20
+ @retry_count = 0
21
21
  @threads = []
22
22
  @running = false
23
23
  @queue = ChimpQueue.instance
24
+ @chimp_queue = Queue.new
24
25
  end
25
-
26
+
26
27
  #
27
28
  # Main entry point for chimpd command line application
28
29
  #
29
30
  def run
30
31
  install_signal_handlers
31
32
  parse_command_line
32
-
33
+
33
34
  puts "chimpd #{VERSION} launching with #{@concurrency} workers"
34
35
  spawn_queue_runner
35
36
  spawn_webserver
37
+ spawn_chimpd_submission_processor
36
38
  run_forever
37
39
  end
38
-
40
+
39
41
  #
40
42
  # Parse chimpd command line options
41
43
  #
@@ -51,7 +53,7 @@ module Chimp
51
53
  [ '--port', '-p', GetoptLong::REQUIRED_ARGUMENT ],
52
54
  [ '--exit', '-x', GetoptLong::NO_ARGUMENT ]
53
55
  )
54
-
56
+
55
57
  opts.each do |opt, arg|
56
58
  case opt
57
59
  when '--logfile', '-l'
@@ -79,52 +81,58 @@ module Chimp
79
81
  puts "Syntax: chimpd [--logfile=<name>] [--concurrency=<c>] [--delay=<d>] [--retry=<r>] [--port=<p>] [--verbose]"
80
82
  exit 1
81
83
  end
82
-
84
+
83
85
  #
84
86
  # Set up logging/verbosity
85
87
  #
86
88
  Chimp.set_verbose(@verbose, @quiet)
87
-
89
+
88
90
  if not @verbose
89
91
  ENV['REST_CONNECTION_LOG'] = "/dev/null"
90
92
  ENV['RESTCLIENT_LOG'] = "/dev/null"
91
93
  end
94
+
95
+ if @quiet
96
+ Log.threshold = Logger::WARN
97
+ end
98
+
92
99
  end
93
-
100
+
94
101
  #
95
102
  # Spawn the ChimpQueue threads
96
103
  #
97
- def spawn_queue_runner
104
+ def spawn_queue_runner
98
105
  @queue.max_threads = @concurrency
99
106
  @queue.delay = @delay
100
107
  @queue.retry_count = @retry_count
101
108
  @queue.start
102
109
  @running = true
103
110
  end
104
-
111
+
105
112
  #
106
113
  # Spawn a WEBrick Web server
107
114
  #
108
115
  def spawn_webserver
109
116
  opts = {
110
- :BindAddress => "localhost",
117
+ :BindAddress => "localhost",
111
118
  :Port => @port,
112
- :MaxClients => 50,
113
- :RequestTimeout => 60
119
+ :MaxClients => 500,
120
+ :RequestTimeout => 120,
121
+ :DoNotReverseLookup => true
114
122
  }
115
-
123
+
116
124
  if not @verbose
117
125
  opts[:Logger] = WEBrick::Log.new("/dev/null")
118
126
  opts[:AccessLog] = [nil, nil]
119
127
  end
120
128
 
121
- @server = ::WEBrick::HTTPServer.new(opts)
129
+ @server = ::WEBrick::HTTPServer.new(opts)
122
130
  @server.mount('/', DisplayServlet)
123
131
  @server.mount('/display', DisplayServlet)
124
132
  @server.mount('/job', JobServlet)
125
133
  @server.mount('/group', GroupServlet)
126
134
  @server.mount('/admin', AdminServlet)
127
-
135
+
128
136
  #
129
137
  # WEBrick threads
130
138
  #
@@ -132,7 +140,7 @@ module Chimp
132
140
  @server.start
133
141
  end
134
142
  end
135
-
143
+
136
144
  #
137
145
  # Process requests forever until we're killed
138
146
  #
@@ -144,18 +152,18 @@ module Chimp
144
152
  end
145
153
  end
146
154
  end
147
-
155
+
148
156
  #
149
157
  # Trap signals to exit cleanly
150
158
  #
151
159
  def install_signal_handlers
152
- ['INT', 'TERM'].each do |signal|
160
+ ['INT', 'TERM'].each do |signal|
153
161
  trap(signal) do
154
162
  self.quit
155
163
  end
156
164
  end
157
165
  end
158
-
166
+
159
167
  #
160
168
  # Quit by waiting for all chimp jobs to finish, not allowing
161
169
  # new jobs on the queue, and killing the web server.
@@ -168,7 +176,51 @@ module Chimp
168
176
  sleep 5
169
177
  exit 0
170
178
  end
171
-
179
+
180
+ #
181
+ # Spawn threads to process submitted requests
182
+ #
183
+ def spawn_chimpd_submission_processor
184
+ n = @concurrency/4
185
+ n = 10 if n < 10
186
+ Log.debug "Logging into API..."
187
+
188
+ #
189
+ # There is a race condition logging in with rest_connection.
190
+ # As a workaround, do a tag query first thing when chimpd starts.
191
+ #
192
+ begin
193
+ c = Chimp.new
194
+ c.interactive = false
195
+ c.quiet = true
196
+ c.tags = ["bogus:tag=true"]
197
+ c.run
198
+ rescue StandardError
199
+ end
200
+
201
+ Log.debug "Spawning #{n} submission processing threads"
202
+ (1..n).each do |n|
203
+ @threads ||=[]
204
+ @threads << Thread.new {
205
+ while true
206
+ begin
207
+ queued_request = @chimp_queue.pop
208
+ group = queued_request.group
209
+ queued_request.interactive = false
210
+
211
+ tasks = queued_request.process
212
+ tasks.each do |task|
213
+ ChimpQueue.instance.push(group, task)
214
+ end
215
+
216
+ rescue StandardError => ex
217
+ Log.error "submission processor: group=\"#{group}\" script=\"#{queued_request.script}\": #{ex}"
218
+ end
219
+ end
220
+ }
221
+ end
222
+ end
223
+
172
224
  #
173
225
  # GenericServlet -- servlet superclass
174
226
  #
@@ -176,13 +228,13 @@ module Chimp
176
228
  def get_verb(req)
177
229
  r = req.request_uri.path.split('/')[2]
178
230
  end
179
-
231
+
180
232
  def get_id(req)
181
233
  uri_parts = req.request_uri.path.split('/')
182
234
  id = uri_parts[-2]
183
235
  return id
184
236
  end
185
-
237
+
186
238
  #
187
239
  # Get the body of the request-- assume YAML
188
240
  #
@@ -194,7 +246,7 @@ module Chimp
194
246
  end
195
247
  end
196
248
  end # GenericServlet
197
-
249
+
198
250
  #
199
251
  # AdminServlet - admin functions
200
252
  #
@@ -206,11 +258,11 @@ module Chimp
206
258
  if shutdown == true
207
259
  ChimpDaemon.instance.quit
208
260
  end
209
-
210
- raise WEBrick::HTTPStatus::OK
261
+
262
+ raise WEBrick::HTTPStatus::OK
211
263
  end
212
- end # AdminServlet
213
-
264
+ end # AdminServlet
265
+
214
266
  #
215
267
  # GroupServlet - group information and control
216
268
  #
@@ -223,17 +275,17 @@ module Chimp
223
275
  #
224
276
  def do_GET(req, resp)
225
277
  jobs = {}
226
-
278
+
227
279
  group_name = req.request_uri.path.split('/')[-2]
228
280
  filter = req.request_uri.path.split('/')[-1]
229
-
281
+
230
282
  g = ChimpQueue[group_name.to_sym]
231
283
  raise WEBrick::HTTPStatus::NotFound, "Group not found" unless g
232
284
  jobs = g.get_jobs_by_status(filter)
233
285
  resp.body = jobs.to_yaml
234
286
  raise WEBrick::HTTPStatus::OK
235
287
  end
236
-
288
+
237
289
  #
238
290
  # POST to a group to trigger a group action
239
291
  # /group/<name>/<action>
@@ -245,21 +297,21 @@ module Chimp
245
297
 
246
298
  if filter == 'create'
247
299
  ChimpQueue.instance.create_group(group_name, payload['type'], payload['concurrency'])
248
-
300
+
249
301
  elsif filter == 'retry'
250
302
  group = ChimpQueue[group_name.to_sym]
251
303
  raise WEBrick::HTTPStatus::NotFound, "Group not found" unless group
252
-
304
+
253
305
  group.requeue_failed_jobs!
254
306
  raise WEBrick::HTTPStatus::OK
255
-
307
+
256
308
  else
257
309
  raise WEBrick::HTTPStatus::PreconditionFailed.new("invalid action")
258
310
  end
259
311
  end
260
-
312
+
261
313
  end
262
-
314
+
263
315
  #
264
316
  # JobServlet - job control
265
317
  #
@@ -273,20 +325,23 @@ module Chimp
273
325
 
274
326
  payload = self.get_payload(req)
275
327
  raise WEBrick::HTTPStatus::PreconditionFailed.new('missing payload') unless payload
276
-
328
+
277
329
  q = ChimpQueue.instance
278
330
  group = payload.group
279
-
331
+
280
332
  #
281
333
  # Ask chimpd to process a Chimp object directly
282
334
  #
335
+ #if verb == 'process' or verb == 'add'
336
+ # payload.interactive = false
337
+ # tasks = payload.process
338
+ # tasks.each do |task|
339
+ # q.push(group, task)
340
+ # end
341
+
283
342
  if verb == 'process' or verb == 'add'
284
- payload.interactive = false
285
- tasks = payload.process
286
- tasks.each do |task|
287
- q.push(group, task)
288
- end
289
-
343
+ ChimpDaemon.instance.chimp_queue.push payload
344
+ id = 0
290
345
  elsif verb == 'update'
291
346
  puts "UPDATE"
292
347
  q.get_job(job_id).status = payload.status
@@ -295,28 +350,28 @@ module Chimp
295
350
  resp.body = {
296
351
  'id' => id
297
352
  }.to_yaml
298
-
353
+
299
354
  raise WEBrick::HTTPStatus::OK
300
355
  end
301
-
356
+
302
357
  def do_GET(req, resp)
303
358
  id = self.get_id(req)
304
359
  verb = self.get_verb(req)
305
360
  job_results = "OK"
306
361
  queue = ChimpQueue.instance
307
-
362
+
308
363
  #
309
364
  # check for special job ids
310
365
  #
311
- jobs = []
366
+ jobs = []
312
367
  jobs << queue.get_job(id.to_i)
313
-
368
+
314
369
  jobs = queue.get_jobs_by_status(:running) if id == 'running'
315
370
  jobs = queue.get_jobs_by_status(:error) if id == 'error'
316
371
  jobs = queue.get_jobs if id == 'all'
317
-
372
+
318
373
  raise WEBrick::HTTPStatus::PreconditionFailed.new('invalid or missing job_id #{id}') unless jobs.size > 0
319
-
374
+
320
375
  #
321
376
  # ACK a job -- mark it as successful even if it failed
322
377
  #
@@ -324,9 +379,9 @@ module Chimp
324
379
  jobs.each do |j|
325
380
  j.status = Executor::STATUS_DONE
326
381
  end
327
-
382
+
328
383
  resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
329
-
384
+
330
385
  #
331
386
  # retry a job
332
387
  #
@@ -334,9 +389,9 @@ module Chimp
334
389
  jobs.each do |j|
335
390
  j.requeue
336
391
  end
337
-
392
+
338
393
  resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
339
-
394
+
340
395
  #
341
396
  # cancel an active job
342
397
  #
@@ -344,9 +399,9 @@ module Chimp
344
399
  jobs.each do |j|
345
400
  j.cancel if j.respond_to? :cancel
346
401
  end
347
-
402
+
348
403
  resp.set_redirect( WEBrick::HTTPStatus::TemporaryRedirect, req.header['referer'])
349
-
404
+
350
405
  #
351
406
  # produce a report
352
407
  #
@@ -355,17 +410,17 @@ module Chimp
355
410
  jobs.each do |j|
356
411
  results << [j.group.group_id, j.class.to_s.sub("Chimp::",""), j.job_id, j.info, j.target, j.time_start, j.time_end, j.get_total_exec_time, j.status].join(",")
357
412
  end
358
-
413
+
359
414
  queue.group.values.each do |g|
360
415
  results << [g.group_id, g.class.to_s.sub("Chimp::",""), "", "", "", g.time_start, g.time_end, g.get_total_exec_time, ""].join(",")
361
416
  end
362
-
417
+
363
418
  job_results = results.join("\n") + "\n"
364
-
419
+
365
420
  resp['Content-type'] = "text/csv"
366
421
  resp['Content-disposition'] = "attachment;filename=chimp.csv"
367
422
  end
368
-
423
+
369
424
  #
370
425
  # return a list of the results
371
426
  #
@@ -373,7 +428,7 @@ module Chimp
373
428
  raise WEBrick::HTTPStatus::OK
374
429
  end
375
430
  end # JobServlet
376
-
431
+
377
432
  #
378
433
  # DisplayServlet
379
434
  #
@@ -387,14 +442,14 @@ module Chimp
387
442
  else
388
443
  template_file_name = 'lib/right_chimp/templates/all_jobs.erb'
389
444
  end
390
-
445
+
391
446
  @template = ERB.new(File.read(template_file_name), nil, ">")
392
447
  end
393
-
394
- queue = ChimpQueue.instance
448
+
449
+ queue = ChimpQueue.instance
395
450
  jobs = queue.get_jobs
396
451
  group_name = nil
397
-
452
+
398
453
  if job_filter == "group"
399
454
  group_name = req.request_uri.path.split('/')[-1]
400
455
  g = ChimpQueue[group_name.to_sym]
@@ -405,12 +460,10 @@ module Chimp
405
460
  count_jobs_queued = queue.get_jobs_by_status(:none).size
406
461
  count_jobs_failed = queue.get_jobs_by_status(:error).size
407
462
  count_jobs_done = queue.get_jobs_by_status(:done).size
408
-
463
+
409
464
  resp.body = @template.result(binding)
410
465
  raise WEBrick::HTTPStatus::OK
411
466
  end
412
467
  end # DisplayServlet
413
- end
468
+ end # ChimpDaemon
414
469
  end
415
-
416
-
@@ -1,6 +1,6 @@
1
1
  module Chimp
2
2
  #
3
- # The ChimpDaemonClient contains the code for communicating with the
3
+ # The ChimpDaemonClient contains the code for communicating with the
4
4
  # RESTful chimpd Web service
5
5
  #
6
6
  class ChimpDaemonClient
@@ -9,23 +9,53 @@ module Chimp
9
9
  #
10
10
  def self.submit(host, port, chimp_object)
11
11
  uri = "http://#{host}:#{port}/job/process"
12
- response = RestClient.post uri, chimp_object.to_yaml
13
-
14
- if response.code > 199 and response.code < 300
15
- begin
12
+ attempts = 3
13
+
14
+ begin
15
+ response = RestClient.post uri, chimp_object.to_yaml
16
+
17
+ if response.code > 199 and response.code < 300
16
18
  id = YAML::load(response.body)['id']
17
- rescue StandardError => ex
18
- puts ex
19
+ return true
20
+ else
21
+ $stderr.puts "WARNING: error submitting to chimpd! response code: #{reponse.code}"
22
+ return false
19
23
  end
20
- else
21
- $stderr.puts "error submitting to chimpd! response code: #{reponse.code}"
24
+
25
+ rescue RestClient::RequestTimeout => ex
26
+ $stderr.puts "WARNING: Request timeout talking to chimpd for job #{chimp_object.script}: #{ex.message} (#{ex.http_code})"
27
+ attempts -= 1
28
+ sleep 5 and retry if attempts > 0
29
+ return false
30
+
31
+ rescue RestClient::InternalServerError => ex
32
+ $stderr.puts "WARNING: Error submitting job to chimpd: #{error_message}, retrying..."
33
+ attempts -= 1
34
+ sleep 5 and retry if attempts > 0
35
+ return false
36
+
37
+ rescue Errno::ECONNRESET => ex
38
+ $stderr.puts "WARNING: Connection reset by peer, retrying..."
39
+ attempts -= 1
40
+ sleep 5 and retry if attempts > 0
41
+ return false
42
+
43
+ rescue Errno::EPIPE => ex
44
+ $stderr.puts "WARNING: broken pipe, retrying..."
45
+ attempts -= 1
46
+ sleep 5 and retry if attempts > 0
47
+ return false
48
+
49
+ rescue Errno::ECONNREFUSED => ex
50
+ $stderr.puts "ERROR: connection refused, aborting"
51
+ return false
52
+
53
+ rescue RestClient::Exception => ex
54
+ $stderr.puts "ERROR: Error submitting job to chimpd #{chimp_object.script}: #{ex.message}"
22
55
  return false
23
56
  end
24
-
25
- puts "chimpd submission complete"
26
- return true
27
57
  end
28
-
58
+
29
59
  #
30
60
  # quit a remote chimpd
31
61
  #
@@ -43,32 +73,32 @@ module Chimp
43
73
  jobs = YAML::load(response.body)
44
74
  return jobs
45
75
  end
46
-
76
+
47
77
  def self.retrieve_group_info(host, port, group_name, status)
48
78
  uri = "http://#{host}:#{port}/group/#{group_name}/#{status}"
49
79
  response = RestClient.get uri
50
80
  group = YAML::load(response.body)
51
81
  return group
52
82
  end
53
-
83
+
54
84
  def self.set_job_status(host, port, job_id, status)
55
85
  uri = "http://#{host}:#{port}/job/#{job_id}/update"
56
86
  response = RestClient.post uri, { :status => status}
57
87
  return YAML::load(response.body)
58
88
  end
59
-
89
+
60
90
  def self.create_group(host, port, name, type, concurrency)
61
91
  uri = "http://#{host}:#{port}/group/#{name}/create"
62
92
  payload = { 'type' => type, 'concurrency' => concurrency }.to_yaml
63
93
  response = RestClient.post(uri, payload)
64
94
  return YAML::load(response.body)
65
95
  end
66
-
96
+
67
97
  def self.retry_group(host, port, group_name)
68
98
  uri = "http://#{host}:#{port}/group/#{group_name}/retry"
69
99
  response = RestClient.post(uri, {})
70
100
  return YAML::load(response.body)
71
101
  end
72
-
102
+
73
103
  end
74
- end
104
+ end
@@ -1,5 +1,5 @@
1
1
  module Chimp
2
-
2
+
3
3
  #
4
4
  # Factory
5
5
  #
@@ -14,7 +14,7 @@ module Chimp
14
14
  end
15
15
  end
16
16
  end
17
-
17
+
18
18
  #
19
19
  # An ExecutionGroup contains a set of Executors to be processed
20
20
  #
@@ -24,7 +24,7 @@ module Chimp
24
24
  class ExecutionGroup
25
25
  attr_accessor :group_id, :description, :concurrency
26
26
  attr_reader :time_start, :time_end
27
-
27
+
28
28
  def initialize(new_group_id=nil)
29
29
  @group_id = new_group_id
30
30
  @queue = []
@@ -34,18 +34,18 @@ module Chimp
34
34
  @time_end = nil
35
35
  @concurrency = 1
36
36
  end
37
-
37
+
38
38
  #
39
39
  # Add something to the work queue
40
- #
40
+ #
41
41
  def push(j)
42
- raise "invalid work" if j == nil
42
+ raise "invalid work" if j == nil
43
43
  j.job_id = IDManager.get if j.job_id == nil
44
44
  j.group = self
45
45
  @queue.push(j)
46
46
  @jobs_by_id[j.job_id] = j
47
47
  end
48
-
48
+
49
49
  #
50
50
  # Take something from the queue
51
51
  #
@@ -54,7 +54,7 @@ module Chimp
54
54
  @time_start = Time.now if @time_start == nil
55
55
  return x
56
56
  end
57
-
57
+
58
58
  #
59
59
  # Return a hash of the results
60
60
  #
@@ -62,7 +62,7 @@ module Chimp
62
62
  return self.get_jobs.map do |task|
63
63
  next if task == nil
64
64
  next if task.server == nil
65
-
65
+
66
66
  {
67
67
  :job_id => task.job_id,
68
68
  :name => task.info,
@@ -76,14 +76,14 @@ module Chimp
76
76
  }
77
77
  end
78
78
  end
79
-
79
+
80
80
  #
81
81
  # Size of the active queue
82
82
  #
83
83
  def size
84
84
  return @queue.size
85
85
  end
86
-
86
+
87
87
  #
88
88
  # Sort queue by server nickname
89
89
  #
@@ -94,35 +94,35 @@ module Chimp
94
94
  end
95
95
  end
96
96
  end
97
-
97
+
98
98
  #
99
99
  # Reset the queue
100
100
  #
101
101
  def reset!
102
102
  @queue = []
103
103
  end
104
-
104
+
105
105
  #
106
106
  # Get all jobs
107
107
  #
108
108
  def get_jobs
109
109
  @jobs_by_id.values
110
110
  end
111
-
111
+
112
112
  #
113
113
  # Get all job ids
114
114
  #
115
115
  def get_job_ids
116
116
  @jobs_by_id.keys
117
117
  end
118
-
118
+
119
119
  #
120
120
  # Get a particular job
121
121
  #
122
122
  def get_job(i)
123
123
  @jobs_by_id[i]
124
124
  end
125
-
125
+
126
126
  #
127
127
  # Get jobs by status
128
128
  #
@@ -133,11 +133,11 @@ module Chimp
133
133
  end
134
134
  return r
135
135
  end
136
-
136
+
137
137
  def job_completed
138
138
  @time_end = Time.now
139
139
  end
140
-
140
+
141
141
  #
142
142
  # Reset all jobs and bulk set them
143
143
  #
@@ -147,7 +147,7 @@ module Chimp
147
147
  self.push(job)
148
148
  end
149
149
  end
150
-
150
+
151
151
  #
152
152
  # An execution group is "ready" if it has work that can be done;
153
153
  # see implementation in child classes.
@@ -155,19 +155,24 @@ module Chimp
155
155
  def ready?
156
156
  raise "unimplemented"
157
157
  end
158
-
158
+
159
159
  #
160
160
  # An execution group is "done" if nothing is queued or running
161
+ # and at least one job has completed.
161
162
  #
162
163
  def done?
163
- return (get_jobs_by_status(Executor::STATUS_NONE).size == 0 && get_jobs_by_status(Executor::STATUS_RUNNING).size == 0)
164
+ return (
165
+ get_jobs_by_status(Executor::STATUS_NONE).size == 0 &&
166
+ get_jobs_by_status(Executor::STATUS_RUNNING).size == 0 &&
167
+ get_jobs_by_status(Executor::STATUS_DONE).size > 0
168
+ )
164
169
  end
165
-
170
+
166
171
  #
167
172
  # Is this execution group running anything?
168
173
  #
169
174
  def running?
170
- total_jobs_running = get_jobs_by_status(Executor::STATUS_NONE).size +
175
+ total_jobs_running = get_jobs_by_status(Executor::STATUS_NONE).size +
171
176
  get_jobs_by_status(Executor::STATUS_RUNNING).size +
172
177
  get_jobs_by_status(Executor::STATUS_RETRYING).size
173
178
  return(total_jobs_running > 0)
@@ -193,7 +198,7 @@ module Chimp
193
198
  job.time_end = nil
194
199
  self.push(job)
195
200
  end
196
-
201
+
197
202
  #
198
203
  # Cancel a job by id
199
204
  #
@@ -205,7 +210,7 @@ module Chimp
205
210
  job.time_end = Time.now
206
211
  @queue.delete(job)
207
212
  end
208
-
213
+
209
214
  #
210
215
  # Return total execution time
211
216
  #
@@ -218,18 +223,18 @@ module Chimp
218
223
  return @time_end.to_i- @time_start.to_i
219
224
  end
220
225
  end
221
-
226
+
222
227
  #
223
228
  # Print out ExecutionGroup information
224
229
  #
225
230
  def to_s
226
231
  return "#{self.class}[#{group_id}]: ready=#{self.ready?} total_jobs=#{@jobs_by_id.size} queued_jobs=#{self.size}"
227
232
  end
228
-
233
+
229
234
  ###################################
230
235
  protected
231
236
  ###################################
232
-
237
+
233
238
  #
234
239
  # Return total execution time or -1 for errors
235
240
  #
@@ -238,7 +243,7 @@ module Chimp
238
243
  end
239
244
 
240
245
  end
241
-
246
+
242
247
  #
243
248
  # SerialExecutionGroup: run only one job at a time
244
249
  #
@@ -246,12 +251,12 @@ module Chimp
246
251
  def ready?
247
252
  return get_jobs_by_status(Executor::STATUS_RUNNING).size == 0 && get_jobs_by_status(Executor::STATUS_NONE).size > 0
248
253
  end
249
-
254
+
250
255
  def short_name
251
256
  "S"
252
257
  end
253
258
  end
254
-
259
+
255
260
  #
256
261
  # ParallelExecutionGroup: run multiple jobs at once
257
262
  #
@@ -260,7 +265,7 @@ module Chimp
260
265
  super(new_group_id)
261
266
  @concurrency = 25
262
267
  end
263
-
268
+
264
269
  #
265
270
  # FIXME - we're not currently using the @concurrency setting to limit execution
266
271
  # due to an unknown bug...
@@ -268,7 +273,7 @@ module Chimp
268
273
  def ready?
269
274
  return (get_jobs_by_status(Executor::STATUS_NONE).size > 0) # and get_jobs_by_status(Executor::STATUS_RUNNING).size < @concurrency)
270
275
  end
271
-
276
+
272
277
  def short_name
273
278
  "P"
274
279
  end
@@ -1,3 +1,3 @@
1
1
  module Chimp
2
- VERSION = "1.0.3"
2
+ VERSION = "1.0.7"
3
3
  end
metadata CHANGED
@@ -1,32 +1,36 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: right_chimp
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.7
5
+ prerelease:
5
6
  platform: ruby
6
7
  authors:
7
8
  - Christopher Deutsch
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2013-06-05 00:00:00.000000000 Z
12
+ date: 2013-06-24 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
- name: progressbar
15
+ name: rest_connection
15
16
  requirement: !ruby/object:Gem::Requirement
17
+ none: false
16
18
  requirements:
17
19
  - - ~>
18
20
  - !ruby/object:Gem::Version
19
- version: 0.11.0
21
+ version: 1.0.10
20
22
  type: :runtime
21
23
  prerelease: false
22
24
  version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
23
26
  requirements:
24
27
  - - ~>
25
28
  - !ruby/object:Gem::Version
26
- version: 0.11.0
29
+ version: 1.0.10
27
30
  - !ruby/object:Gem::Dependency
28
31
  name: rest-client
29
32
  requirement: !ruby/object:Gem::Requirement
33
+ none: false
30
34
  requirements:
31
35
  - - ~>
32
36
  - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
34
38
  type: :runtime
35
39
  prerelease: false
36
40
  version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
37
42
  requirements:
38
43
  - - ~>
39
44
  - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
41
46
  - !ruby/object:Gem::Dependency
42
47
  name: rake
43
48
  requirement: !ruby/object:Gem::Requirement
49
+ none: false
44
50
  requirements:
45
51
  - - ~>
46
52
  - !ruby/object:Gem::Version
@@ -48,27 +54,47 @@ dependencies:
48
54
  type: :runtime
49
55
  prerelease: false
50
56
  version_requirements: !ruby/object:Gem::Requirement
57
+ none: false
51
58
  requirements:
52
59
  - - ~>
53
60
  - !ruby/object:Gem::Version
54
61
  version: 0.9.2.2
55
62
  - !ruby/object:Gem::Dependency
56
- name: rest_connection
63
+ name: nokogiri
57
64
  requirement: !ruby/object:Gem::Requirement
65
+ none: false
58
66
  requirements:
59
67
  - - ~>
60
68
  - !ruby/object:Gem::Version
61
- version: 1.0.6
69
+ version: 1.5.9
62
70
  type: :runtime
63
71
  prerelease: false
64
72
  version_requirements: !ruby/object:Gem::Requirement
73
+ none: false
65
74
  requirements:
66
75
  - - ~>
67
76
  - !ruby/object:Gem::Version
68
- version: 1.0.6
77
+ version: 1.5.9
78
+ - !ruby/object:Gem::Dependency
79
+ name: progressbar
80
+ requirement: !ruby/object:Gem::Requirement
81
+ none: false
82
+ requirements:
83
+ - - ~>
84
+ - !ruby/object:Gem::Version
85
+ version: 0.11.0
86
+ type: :runtime
87
+ prerelease: false
88
+ version_requirements: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ~>
92
+ - !ruby/object:Gem::Version
93
+ version: 0.11.0
69
94
  - !ruby/object:Gem::Dependency
70
95
  name: rspec
71
96
  requirement: !ruby/object:Gem::Requirement
97
+ none: false
72
98
  requirements:
73
99
  - - ~>
74
100
  - !ruby/object:Gem::Version
@@ -76,6 +102,7 @@ dependencies:
76
102
  type: :development
77
103
  prerelease: false
78
104
  version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
79
106
  requirements:
80
107
  - - ~>
81
108
  - !ruby/object:Gem::Version
@@ -127,26 +154,27 @@ files:
127
154
  - spec/spec_selection.rb
128
155
  homepage: https://github.com/rightscale/right_chimp
129
156
  licenses: []
130
- metadata: {}
131
157
  post_install_message:
132
158
  rdoc_options: []
133
159
  require_paths:
134
160
  - lib
135
161
  required_ruby_version: !ruby/object:Gem::Requirement
162
+ none: false
136
163
  requirements:
137
- - - '>='
164
+ - - ! '>='
138
165
  - !ruby/object:Gem::Version
139
166
  version: '0'
140
167
  required_rubygems_version: !ruby/object:Gem::Requirement
168
+ none: false
141
169
  requirements:
142
- - - '>='
170
+ - - ! '>='
143
171
  - !ruby/object:Gem::Version
144
172
  version: '0'
145
173
  requirements: []
146
174
  rubyforge_project:
147
- rubygems_version: 2.0.3
175
+ rubygems_version: 1.8.23
148
176
  signing_key:
149
- specification_version: 4
177
+ specification_version: 3
150
178
  summary: RightScale platform command-line tool
151
179
  test_files:
152
180
  - spec/spec_chimp.rb
checksums.yaml DELETED
@@ -1,7 +0,0 @@
1
- ---
2
- SHA1:
3
- metadata.gz: 1562feef06b0bcc0b1c87bd55c8421abafd7ca26
4
- data.tar.gz: 724add71f798b34e6a4a4e37e2611c90d60e44b0
5
- SHA512:
6
- metadata.gz: 5aee4d9a399b1a0b7dc99b0b05640bb372ced0015a54f61caff3f9e7b0ab01186be690d94ffa5a0c1eaeb4fa2e8f5f07c68ddb547561630cfe4a5d15953c21bb
7
- data.tar.gz: 6d1ec02570a46c928512df6b0387759faad9a15b7774d4ed408a5fd14f693da8d1d5af97cbe8fc5d35eb5d052062d689ea60963f2d664979a4fdea3b50f3da58