mesa_test 0.0.20 → 0.0.21

Sign up to get free protection for your applications and to get access to all the features.
Files changed (4) hide show
  1. checksums.yaml +5 -5
  2. data/bin/mesa_test +181 -4
  3. data/lib/mesa_test.rb +223 -11
  4. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: ae90768598e2d3b36463137ab145993b1ba1c541
4
- data.tar.gz: 1166777ede80f334d8f19f2eb435a8ee4685445b
2
+ SHA256:
3
+ metadata.gz: 959837654b03c806d656305dbbf32b850a155b85e17df6c81306d96483b91573
4
+ data.tar.gz: d690529aa2e9b54fc8cee25753cf6270a3b7adabec2c0879f84b3930103df54a
5
5
  SHA512:
6
- metadata.gz: 97678b777bff66fb193fb921bc3d75a713f5b83660b68db596a59c698f54c842e2b0cd35bc796d7b6f21e737d7a8594e83f586c0ce0b8c8401d8bd064dc7c31f
7
- data.tar.gz: 14a794c85800887f0e49e2c08da7cf02ad0fa68f5f5a8a81c63495949c425486822628d86511a168c99460315c4bd89fe0a660afdfc5ce7e872954c63d682ecc
6
+ metadata.gz: 0ea095803d606961091d77e13a0cd406fbec36d962d70abb853023e334fcc294c7c0cfc24ca4f166c51959094832c491622ec5a3f621348d15df0c3789adb27b
7
+ data.tar.gz: 9f313f3c236fee9cd94f51bb658d04103de95d0c6d1d9c0f2d70eb504bdbe9719025c020cfbe7868f05727bc32f1612eedd36f41f4a2426824b4817306583511
data/bin/mesa_test CHANGED
@@ -101,16 +101,22 @@ class MesaTest < Thor
101
101
 
102
102
  With --submit option, upload results to MESATestHub. By default, this is on.
103
103
  To run without submission, use --no-submit.
104
+
105
+ With --svn option, confirm version number with svn rather than trusting
106
+ self-reported version number. By default, this is on. To run without using
107
+ svn, use --no-svn.
104
108
  LONGDESC
105
109
  option :diff, type: :boolean, default: false
106
110
  option :force, type: :boolean, aliases: '-f'
107
111
  option :log, type: :boolean, default: true
108
112
  option :module, type: :string, default: :all
109
113
  option :submit, type: :boolean, default: true
114
+ option :svn, type: :boolean, default: true
110
115
  def test_all(mesa_dir)
111
116
  s = create_and_check_submitter(force: options[:force]) if options[:submit]
112
117
  m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
113
- mod: options[:module].downcase.to_sym)
118
+ mod: options[:module].downcase.to_sym,
119
+ use_svn: options[:svn])
114
120
 
115
121
  # run all tests
116
122
  m.each_test_run_and_diff(log_results: options[:log],
@@ -120,6 +126,55 @@ class MesaTest < Thor
120
126
  s.submit_all(m, options[:module].downcase.to_sym) if options[:submit]
121
127
  end
122
128
 
129
+ desc 'test_revision MESA_DIR', 'test and submit all tests as a packaged version'
130
+ long_desc <<-LONGDESC
131
+ Run and check all test cases residing in MESA_DIR/star/test_suite. Then
132
+ report results to MesaTestHub. Specifically, runs and checks all tests
133
+ detailed in MESA_DIR/star/test_suite/do1_test_source. Also submit version
134
+ information with test instances. Submits all test instances and version data
135
+ in a single request rather than case-by-case in `test_all`.
136
+
137
+ With --diff option, assume the checksums are up-to-date (behave like
138
+ each_test_run_and_diff), off by default.
139
+
140
+ With --force option, skip confirmation of computer details, assuming values
141
+ in ~/.mesa_test.yml are correct. Only relevant if --submit option is on
142
+ (by default it is).
143
+
144
+ With --module option, select which module to search through, with the default
145
+ being "all" (search all modules in order). Example: --module=star.
146
+
147
+ With --log option, save yml file of test results in test directory and a
148
+ summary in the test suite directory. On by default. Shut off with --no-log.
149
+
150
+ With --submit option, upload results to MESATestHub. By default, this is on.
151
+ To run without submission, use --no-submit.
152
+
153
+ With --svn option, get version info (number, author, commit message) from
154
+ svn. By default, this is on. To shut off, use --no-svn. If off, no author or
155
+ commit message are saved.
156
+ LONGDESC
157
+ option :diff, type: :boolean, default: false
158
+ option :force, type: :boolean, aliases: '-f'
159
+ option :log, type: :boolean, default: true
160
+ option :module, type: :string, default: :all
161
+ option :submit, type: :boolean, default: true
162
+ option :svn, type: :boolean, default: true
163
+ def test_revision(mesa_dir)
164
+ s = create_and_check_submitter(force: options[:force]) if options[:submit]
165
+ m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
166
+ mod: options[:module].downcase.to_sym,
167
+ use_svn: options[:svn])
168
+
169
+ # run all tests
170
+ m.each_test_run_and_diff(log_results: options[:log],
171
+ mod: options[:module].downcase.to_sym)
172
+
173
+ # submit all tests
174
+ s.submit_revision(m) if options[:submit]
175
+ end
176
+
177
+
123
178
  desc 'submit_one MESA_DIR TEST_CASE', 'submit one completed test case'
124
179
  long_desc <<-LONGDESC
125
180
  Load complete test case data from existing YAML file and submit to MESA.
@@ -163,13 +218,19 @@ class MesaTest < Thor
163
218
 
164
219
  With --module option, select which module to search through, with the default
165
220
  being "all" (search all modules in order). Example: --module=star.
221
+
222
+ With --svn option, confirm version number with svn rather than trusting
223
+ self-reported version number. By default, this is on. To run without using
224
+ svn, use --no-svn.
166
225
  LONGDESC
167
226
  option :force, type: :boolean, aliases: '-f'
168
227
  option :module, type: :string, aliases: '-m', default: 'all'
228
+ option :svn, type: :boolean, default: true
169
229
  def submit_all(mesa_dir)
170
230
  s = create_and_check_submitter(force: options[:force])
171
231
  m = create_and_check_mesa(mesa_dir: mesa_dir,
172
- mod: options[:module].downcase.to_sym)
232
+ mod: options[:module].downcase.to_sym,
233
+ use_svn: options[:svn])
173
234
 
174
235
  # load all test data
175
236
  m.each_test_load_results(mod: options[:module].downcase.to_sym)
@@ -178,6 +239,42 @@ class MesaTest < Thor
178
239
  s.submit_all(m, options[:module].downcase.to_sym)
179
240
  end
180
241
 
242
+ desc 'submit_revision MESA_DIR', 'submit version with pre-run test cases'
243
+ long_desc <<-LONGDESC
244
+ Load results for all test cases residing in MESA_DIR/star/test_suite. Then
245
+ submit results to MesaTestHub. Specifically, loads and submits all tests
246
+ detailed in MESA_DIR/star/test_suite/do1_test_source along with version data
247
+ (number, author, and commit message). Unlike submit_all, this submits all
248
+ test cases AND version info in a single request rather than just the cases
249
+ in series.
250
+
251
+ With --force option, skip confirmation of computer details, assuming values
252
+ in ~/.mesa_test.yml are correct.
253
+
254
+ With --module option, select which module to search through, with the default
255
+ being "all" (search all modules in order). Example: --module=star.
256
+
257
+ With --svn option, get version info (number, author, commit message) from
258
+ svn. By default, this is on. To shut off, use --no-svn. If off, no author or
259
+ commit message are submitted.
260
+ LONGDESC
261
+ option :force, type: :boolean, aliases: '-f'
262
+ option :module, type: :string, aliases: '-m', default: 'all'
263
+ option :svn, type: :boolean, default: true
264
+ def submit_revision(mesa_dir)
265
+ s = create_and_check_submitter(force: options[:force])
266
+ m = create_and_check_mesa(mesa_dir: mesa_dir,
267
+ mod: options[:module].downcase.to_sym,
268
+ use_svn: options[:svn])
269
+
270
+ # load all test data
271
+ m.each_test_load_results(mod: options[:module].downcase.to_sym)
272
+
273
+ # submit all tests
274
+ s.submit_revision(m)
275
+ end
276
+
277
+
181
278
  desc 'install VERSION_NUMBER MESA_DIR', 'download and install mesa release '\
182
279
  'VERSION_NUMBER to directory MESA_DIR'
183
280
  long_desc <<-LONGDESC
@@ -262,6 +359,86 @@ class MesaTest < Thor
262
359
  m.destroy if successfully_submitted && options[:destroy]
263
360
  end
264
361
 
362
+ desc 'install_and_test_revision VERSION_NUMBER MESA_DIR', 'download and '\
363
+ 'install mesa release VERSION_NUMBER to directory MESA_DIR, run all tests,'\
364
+ 'and submit version and test data in single request.'
365
+ long_desc <<-LONGDESC
366
+ Calls to svn to install mesa release VERSION_NUMBER into the directory
367
+ MESA_DIR. Basically just an svn checkout followed by going into the directory
368
+ and running ./clean and ./install. SDK or compilers must be set up prior.
369
+ Once installation is complete, run the test suite, and report results
370
+ (version and all test data) to MesaTestHub in a single request (whereas
371
+ install_and_test_all only submits test data and does so in a serial fashion).
372
+ Does not affect the user's MESA_DIR or other environment variables. This is
373
+ basically a shortcut for running
374
+
375
+ `mesa_test install SOME_VERSION SOME_DIR`
376
+
377
+ followed by
378
+
379
+ `mesa_test test_revision SOME_DIR`
380
+
381
+ If VERSION_NUMBER is 'non-paxton' or 'non_paxton', then the most recent
382
+ commit by anyone other than Bill Paxton ('bill_paxton') is tested and the
383
+ --diff flag is ignored and set to false. Note that using 'HEAD' for version
384
+ number will automatically select the most recent release, regardless of the
385
+ author.
386
+
387
+ With --diff option, assume the checksums are up-to-date (behave like
388
+ each_test_run_and_diff), off by default.
389
+
390
+ Use flag --destroy to self destruct MESA_DIR after successful test
391
+ submission. Essentially does rm -rf MESA_DIR after the test suite. Useful
392
+ for automated testing without piling up disk space.
393
+
394
+ Use flag --force to skip confirmation of computer details if they can be
395
+ read from ~/.mesa_test.yml.
396
+
397
+ With --log option, save yml file of test results in test directory and a
398
+ summary in the test suite directory. On by default. Shut off with --no-log.
399
+
400
+ With --module option, select which module to search through, with the default
401
+ being "all". Example: --module=star.
402
+
403
+ With --svn option, get version info (number, author, commit message) from
404
+ svn. By default, this is on. To shut off, use --no-svn. If off, no author or
405
+ commit message are submitted.
406
+ LONGDESC
407
+ option :diff, type: :boolean, default: false
408
+ option :destroy, type: :boolean
409
+ option :log, type: :boolean, default: true
410
+ option :force, type: :boolean, aliases: '-f'
411
+ option :module, type: :string, aliases: '-m', default: 'all'
412
+ option :svn, type: :boolean, default: true
413
+ def install_and_test_all(version, mesa_dir)
414
+ s = create_and_check_submitter(force: options[:force])
415
+ mesa_version, diff = filter_version_and_diff(version, s)
416
+ if mesa_version.nil?
417
+ puts 'No commits by anyone other than Bill Paxton since '\
418
+ "r#{s.last_tested}. Exiting."
419
+ return
420
+ end
421
+ m = Mesa.download(version_number: mesa_version, new_mesa_dir: mesa_dir,
422
+ use_svn: options[:svn])
423
+ m.clean
424
+ m.install
425
+
426
+ m = create_and_check_mesa(mesa_dir: mesa_dir, diff: diff,
427
+ mod: options[:module].downcase.to_sym,
428
+ use_svn: options[:svn])
429
+
430
+ # run all tests. Don't be fooled, whether or not a diff happens is set
431
+ # by the diff attribute of the mesa object, set above.
432
+ m.each_test_run_and_diff(mod: options[:module].downcase.to_sym,
433
+ log_results: options[:log])
434
+
435
+ # submit all tests
436
+ successfully_submitted = s.submit_revision(m)
437
+
438
+ # if requested, and if submission successful, destroy the directory
439
+ m.destroy if successfully_submitted && options[:destroy]
440
+ end
441
+
265
442
  private
266
443
 
267
444
  def check_user_and_computer(submitter)
@@ -287,8 +464,8 @@ class MesaTest < Thor
287
464
  s
288
465
  end
289
466
 
290
- def create_and_check_mesa(mesa_dir:, diff: true, mod: :all)
291
- m = Mesa.new mesa_dir: mesa_dir
467
+ def create_and_check_mesa(mesa_dir:, diff: true, mod: :all, use_svn: true)
468
+ m = Mesa.new mesa_dir: mesa_dir, use_svn: use_svn
292
469
  unless m.installed?
293
470
  raise MesaDirError, "Invalid MESA_DIR: #{mesa_dir}. Please download " \
294
471
  'and install a valid MESA version or provide the path to one.'
data/lib/mesa_test.rb CHANGED
@@ -246,7 +246,11 @@ e-mail and password will be stored in plain text.'
246
246
  platform_version: platform_version,
247
247
  omp_num_threads: test_case.test_omp_num_threads,
248
248
  success_type: test_case.success_type,
249
- failure_type: test_case.failure_type
249
+ failure_type: test_case.failure_type,
250
+ steps: test_case.steps,
251
+ retries: test_case.retries,
252
+ backups: test_case.backups,
253
+ summary_text: test_case.summary_text
250
254
  }
251
255
 
252
256
  # enter in test-specific data, DISABLED FOR NOW
@@ -258,6 +262,57 @@ e-mail and password will be stored in plain text.'
258
262
  res
259
263
  end
260
264
 
265
+ def revision_submit_params(mesa)
266
+ mesa.load_svn_data if mesa.use_svn?
267
+ # version gives data about version
268
+ # user gives data about the user and computer submitting information
269
+ # instances is array of hashes that identify test instances (more below)
270
+ res = {
271
+ version: {number: mesa.version_number},
272
+ user: {email: email, password: password, computer: computer_name},
273
+ instances: []
274
+ }
275
+ if mesa.use_svn?
276
+ res[:version][:author] = mesa.svn_author
277
+ res[:version][:log] = mesa.svn_log
278
+ end
279
+
280
+ # hold on to test case names that fail in synthesizing params
281
+ has_errors = []
282
+
283
+ # each instance has basic information in :test_instance and extra
284
+ # information that requires the web app to work, stored in :extra
285
+ mesa.test_names.each do |mod, names|
286
+ names.each do |test_name|
287
+ begin
288
+ test_case = mesa.test_cases[mod][test_name]
289
+ res[:instances] << {
290
+ test_instance: {
291
+ runtime_seconds: test_case.runtime_seconds,
292
+ passed: test_case.passed?,
293
+ compiler: compiler,
294
+ compiler_version: compiler_version,
295
+ platform_version: platform_version,
296
+ omp_num_threads: test_case.test_omp_num_threads,
297
+ success_type: test_case.success_type,
298
+ failure_type: test_case.failure_type,
299
+ steps: test_case.steps,
300
+ retries: test_case.retries,
301
+ backups: test_case.backups,
302
+ summary_text: test_case.summary_text
303
+ },
304
+ extra: { test_case: test_name, mod: mod }
305
+ }
306
+ rescue TestCaseDirError
307
+ shell.say "Passage status for #{test_case.test_name} not yet "\
308
+ 'known. Run test first and then submit.', :red
309
+ has_errors.append(test_case)
310
+ end
311
+ end
312
+ end
313
+ [res, has_errors]
314
+ end
315
+
261
316
  def confirm_computer
262
317
  uri = URI.parse(base_uri + '/check_computer.json')
263
318
  https = Net::HTTP.new(uri.hostname, uri.port)
@@ -271,7 +326,6 @@ e-mail and password will be stored in plain text.'
271
326
  password: password,
272
327
  computer_name: computer_name
273
328
  }.to_json
274
-
275
329
  JSON.parse(https.request(request).body).to_hash
276
330
  end
277
331
 
@@ -351,15 +405,53 @@ e-mail and password will be stored in plain text.'
351
405
  # SUBMITTED (irrespective of passing status)
352
406
  success
353
407
  end
408
+
409
+ # similar to submit_all, but does EVERYTHING in one post, including
410
+ # version information. No support for individual modules now.
411
+ def submit_revision(mesa)
412
+ uri = URI.parse(base_uri + '/versions/submit_revision.json')
413
+ https = Net::HTTP.new(uri.hostname, uri.port)
414
+ https.use_ssl = true if base_uri.include? 'https'
415
+
416
+ request = Net::HTTP::Post.new(
417
+ uri,
418
+ initheader = { 'Content-Type' => 'application/json' }
419
+ )
420
+ request_data, error_cases = revision_submit_params(mesa)
421
+ if request_data[:instances].empty?
422
+ shell.say "No completed test data found in #{mesa.mesa_dir}. Aborting.",
423
+ :red
424
+ return false
425
+ end
426
+ request.body = request_data.to_json
427
+
428
+ # verbose = true
429
+ # puts "\n" if verbose
430
+ # puts JSON.parse(request.body).to_hash if verbose
431
+
432
+ response = https.request request
433
+ # puts JSON.parse(response.body).to_hash if verbose
434
+ if !response.is_a? Net::HTTPCreated
435
+ shell.say "\nFailed to submit some or all cases and/or version data.",
436
+ :red
437
+ false
438
+ elsif !error_cases.empty?
439
+ shell.say "\nFailed to gather data for the following cases:", :red
440
+ error_cases.each { |tc| shell.say " #{tc.test_name}", :red }
441
+ false
442
+ end
443
+ true
444
+ end
354
445
  end
355
446
 
356
447
  class Mesa
357
448
  SVN_URI = 'svn://svn.code.sf.net/p/mesa/code/trunk'.freeze
358
449
 
359
- attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell
450
+ attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell,
451
+ :svn_version, :svn_author, :svn_log
360
452
  attr_accessor :update_checksums
361
453
 
362
- def self.download(version_number: nil, new_mesa_dir: nil)
454
+ def self.download(version_number: nil, new_mesa_dir: nil, use_svn: true)
363
455
  new_mesa_dir ||= File.join(ENV['HOME'], 'mesa-test-r' + version_number.to_s)
364
456
  success = bash_execute(
365
457
  "svn co -r #{version_number} " \
@@ -369,7 +461,7 @@ class Mesa
369
461
  raise MesaDirError, 'Encountered a problem in download mesa ' \
370
462
  "revision #{version_number}."
371
463
  end
372
- Mesa.new(mesa_dir: new_mesa_dir)
464
+ Mesa.new(mesa_dir: new_mesa_dir, use_svn: use_svn)
373
465
  end
374
466
 
375
467
  def self.log_since(last_tested = DEFAULT_REVISION)
@@ -418,8 +510,9 @@ class Mesa
418
510
  nil
419
511
  end
420
512
 
421
- def initialize(mesa_dir: ENV['MESA_DIR'])
513
+ def initialize(mesa_dir: ENV['MESA_DIR'], use_svn: true)
422
514
  @mesa_dir = mesa_dir
515
+ @use_svn = use_svn
423
516
  @update_checksums = false
424
517
 
425
518
  # these get populated by calling #load_test_data
@@ -429,16 +522,39 @@ class Mesa
429
522
 
430
523
  # way to output colored text
431
524
  @shell = Thor::Shell::Color.new
525
+
526
+ # these can be populated by calling load_svn_data
527
+ @svn_version = nil
528
+ @svn_author = nil
529
+ @svn_log = nil
530
+ load_svn_data if use_svn?
531
+ end
532
+
533
+ def use_svn?
534
+ @use_svn
432
535
  end
433
536
 
434
537
  def version_number
435
- # prefer svn's reported version number
436
- version = svn_version_number
538
+ version = @svn_version || 0
437
539
  # fall back to MESA_DIR/data's version number svn didn't work
438
540
  version = data_version_number unless version > 0
439
541
  version
440
542
  end
441
543
 
544
+ def log_entry
545
+ `svn log #{SVN_URI} -r #{version_number}`
546
+ end
547
+
548
+ def load_svn_data
549
+ # if this number is bad, #version_number will use fallback method
550
+ @svn_version = svn_version_number
551
+ lines = log_entry.split("\n").reject { |line| line =~ /^-+$/ or line.empty?}
552
+ data_line = lines.shift
553
+ revision, author, date, length = data_line.split('|')
554
+ @svn_author = author.strip
555
+ @svn_log = lines.join("\n").strip
556
+ end
557
+
442
558
  # get version number from svn (preferred method)
443
559
  def svn_version_number
444
560
  return `svnversion #{mesa_dir}`.strip.to_i
@@ -728,7 +844,8 @@ end
728
844
  class MesaTestCase
729
845
  attr_reader :test_name, :mesa_dir, :mesa, :success_string, :final_model,
730
846
  :failure_msg, :success_msg, :photo, :runtime_seconds,
731
- :test_omp_num_threads, :mesa_version, :shell, :mod
847
+ :test_omp_num_threads, :mesa_version, :shell, :mod, :retries,
848
+ :backups, :steps, :runtime_minutes, :summary_text
732
849
  attr_accessor :data_names, :data_types, :failure_type, :success_type,
733
850
  :outcome
734
851
 
@@ -750,6 +867,11 @@ class MesaTestCase
750
867
  @outcome = :not_tested
751
868
  @runtime_seconds = 0
752
869
  @test_omp_num_threads = 1
870
+ @runtime_minutes = 0
871
+ @retries = 0
872
+ @backups = 0
873
+ @steps = 0
874
+ @summary_text = ''
753
875
  unless MesaTestCase.modules.include? mod
754
876
  raise TestCaseDirError, "Invalid module: #{mod}. Must be one of: " +
755
877
  MesaTestCase.modules.join(', ')
@@ -794,7 +916,7 @@ class MesaTestCase
794
916
  false
795
917
  else
796
918
  raise TestCaseDirError, 'Cannot determine pass/fail status of ' \
797
- '#{test_name} yet.'
919
+ "#{test_name} yet."
798
920
  end
799
921
  end
800
922
 
@@ -886,7 +1008,12 @@ class MesaTestCase
886
1008
  'outcome' => outcome,
887
1009
  'omp_num_threads' => test_omp_num_threads,
888
1010
  'success_type' => success_type,
889
- 'failure_type' => failure_type
1011
+ 'failure_type' => failure_type,
1012
+ 'runtime_minutes' => runtime_minutes,
1013
+ 'retries' => retries,
1014
+ 'backups' => backups,
1015
+ 'steps' => steps,
1016
+ 'summary_text' => summary_text
890
1017
  }
891
1018
  File.open(save_file, 'w') { |f| f.write(YAML.dump(res)) }
892
1019
  shell.say "Successfully saved results to file #{save_file}.\n", :green
@@ -909,6 +1036,11 @@ class MesaTestCase
909
1036
  @test_omp_num_threads = data['omp_num_threads'] || @test_omp_num_threads
910
1037
  @success_type = data['success_type'] || @success_type
911
1038
  @failure_type = data['failure_type'] || @failure_type
1039
+ @runtime_minutes = data['runtime_minutes'] || @runtime_minutes
1040
+ @retries = data['retries'] || @retries
1041
+ @backups = data['backups'] || @backups
1042
+ @steps = data['steps'] || @steps
1043
+ @summary_text = data['summary_text'] || @summary_text
912
1044
 
913
1045
  # convert select data to symbols since that is how they are used
914
1046
  @outcome = @outcome.to_sym if @outcome
@@ -918,6 +1050,31 @@ class MesaTestCase
918
1050
  shell.say "Done loading data from #{load_file}.\n", :green
919
1051
  end
920
1052
 
1053
+ def load_summary_data
1054
+ out_data = parse_out
1055
+ @runtime_minutes = out_data[:runtime_minutes]
1056
+ @retries = out_data[:retries]
1057
+ @backups = out_data[:backups]
1058
+ @steps = out_data[:steps]
1059
+ @summary_text = get_summary_text
1060
+ end
1061
+
1062
+ def parse_out
1063
+ runtime_minutes = 0
1064
+ retries = 0
1065
+ backups = 0
1066
+ steps = 0
1067
+ run_summaries.each do |summary|
1068
+ summary =~ /^\s*runtime\s*\(minutes\),\s+retries,\s+backups,\ssteps\s+(\d+\.?\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/
1069
+ runtime_minutes += $1.to_f
1070
+ retries += $2.to_i
1071
+ backups += $3.to_i
1072
+ steps += $4.to_i
1073
+ end
1074
+ {runtime_minutes: runtime_minutes, retries: retries, backups: backups,
1075
+ steps: steps}
1076
+ end
1077
+
921
1078
  private
922
1079
 
923
1080
  def data_types
@@ -1085,6 +1242,9 @@ class MesaTestCase
1085
1242
 
1086
1243
  # only check restart/photo if we get through run successfully
1087
1244
  check_restart if check_run
1245
+
1246
+ # get reported runtime, retries, backups, and steps
1247
+ load_summary_data
1088
1248
  end
1089
1249
 
1090
1250
  # append contents of err.txt to end of out.txt, then delete err.txt
@@ -1130,6 +1290,58 @@ class MesaTestCase
1130
1290
  return unless File.exist?(final_model)
1131
1291
  FileUtils.rm(final_model)
1132
1292
  end
1293
+
1294
+ # helpers for getting run summaries
1295
+ def run_summaries
1296
+ # look at all lines in out.txt
1297
+ lines = IO.readlines(File.join(test_case_dir, 'out.txt'))
1298
+
1299
+ # find lines with summary information
1300
+ summary_line_numbers = []
1301
+ lines.each_with_index do |line, i|
1302
+ if line =~ /^\s*runtime \(minutes\),\s+retries,\s+backups,\ssteps/
1303
+ summary_line_numbers << i
1304
+ end
1305
+ end
1306
+
1307
+ # find lines indicating passage or failure of runs and restarts
1308
+ run_finish_line_numbers = []
1309
+ restart_finish_line_numbers = []
1310
+ lines.each_with_index do |line, i|
1311
+ if line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+restart/
1312
+ restart_finish_line_numbers << i
1313
+ elsif line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+run/
1314
+ run_finish_line_numbers << i
1315
+ end
1316
+ end
1317
+
1318
+ # only keep summaries that correspond to runs rather than restart
1319
+ summary_line_numbers.select do |i|
1320
+ run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
1321
+ end.map { |line_number| lines[line_number] }
1322
+ end
1323
+
1324
+ def get_summary_text
1325
+ lines = IO.readlines(File.join(test_case_dir, 'out.txt')).select do |line|
1326
+ line =~ /^\s*runtime/
1327
+ end.join
1328
+ end
1329
+
1330
+ def run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
1331
+ # iterate from starting line (a summary line) up to largest PASS/FAIL
1332
+ # line, bail out if summary line is beyond any PASS/FAIL line
1333
+ max_line = run_finish_line_numbers.max || 0
1334
+ max_line = [max_line, (restart_finish_line_numbers.max || 0)].max
1335
+ return false if i > max_line
1336
+ # return true if next PASS/FAIL line is for a run and fail if it is for a
1337
+ # restart
1338
+ i.upto(max_line) do |j|
1339
+ return true if run_finish_line_numbers.include?(j)
1340
+ return false if restart_finish_line_numbers.include?(j)
1341
+ end
1342
+ false
1343
+ end
1344
+
1133
1345
  end
1134
1346
 
1135
1347
  ################################
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mesa_test
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.20
4
+ version: 0.0.21
5
5
  platform: ruby
6
6
  authors:
7
7
  - William Wolf
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-12-12 00:00:00.000000000 Z
11
+ date: 2017-01-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: json
@@ -82,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
82
82
  version: '0'
83
83
  requirements: []
84
84
  rubyforge_project:
85
- rubygems_version: 2.6.14
85
+ rubygems_version: 2.7.4
86
86
  signing_key:
87
87
  specification_version: 4
88
88
  summary: Command line tool for running and reporting the MESA test suites.