mesa_test 0.0.20 → 0.0.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. checksums.yaml +5 -5
  2. data/bin/mesa_test +181 -4
  3. data/lib/mesa_test.rb +223 -11
  4. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: ae90768598e2d3b36463137ab145993b1ba1c541
4
- data.tar.gz: 1166777ede80f334d8f19f2eb435a8ee4685445b
2
+ SHA256:
3
+ metadata.gz: 959837654b03c806d656305dbbf32b850a155b85e17df6c81306d96483b91573
4
+ data.tar.gz: d690529aa2e9b54fc8cee25753cf6270a3b7adabec2c0879f84b3930103df54a
5
5
  SHA512:
6
- metadata.gz: 97678b777bff66fb193fb921bc3d75a713f5b83660b68db596a59c698f54c842e2b0cd35bc796d7b6f21e737d7a8594e83f586c0ce0b8c8401d8bd064dc7c31f
7
- data.tar.gz: 14a794c85800887f0e49e2c08da7cf02ad0fa68f5f5a8a81c63495949c425486822628d86511a168c99460315c4bd89fe0a660afdfc5ce7e872954c63d682ecc
6
+ metadata.gz: 0ea095803d606961091d77e13a0cd406fbec36d962d70abb853023e334fcc294c7c0cfc24ca4f166c51959094832c491622ec5a3f621348d15df0c3789adb27b
7
+ data.tar.gz: 9f313f3c236fee9cd94f51bb658d04103de95d0c6d1d9c0f2d70eb504bdbe9719025c020cfbe7868f05727bc32f1612eedd36f41f4a2426824b4817306583511
data/bin/mesa_test CHANGED
@@ -101,16 +101,22 @@ class MesaTest < Thor
101
101
 
102
102
  With --submit option, upload results to MESATestHub. By default, this is on.
103
103
  To run without submission, use --no-submit.
104
+
105
+ With --svn option, confirm version number with svn rather than trusting
106
+ self-reported version number. By default, this is on. To run without using
107
+ svn, use --no-svn.
104
108
  LONGDESC
105
109
  option :diff, type: :boolean, default: false
106
110
  option :force, type: :boolean, aliases: '-f'
107
111
  option :log, type: :boolean, default: true
108
112
  option :module, type: :string, default: :all
109
113
  option :submit, type: :boolean, default: true
114
+ option :svn, type: :boolean, default: true
110
115
  def test_all(mesa_dir)
111
116
  s = create_and_check_submitter(force: options[:force]) if options[:submit]
112
117
  m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
113
- mod: options[:module].downcase.to_sym)
118
+ mod: options[:module].downcase.to_sym,
119
+ use_svn: options[:svn])
114
120
 
115
121
  # run all tests
116
122
  m.each_test_run_and_diff(log_results: options[:log],
@@ -120,6 +126,55 @@ class MesaTest < Thor
120
126
  s.submit_all(m, options[:module].downcase.to_sym) if options[:submit]
121
127
  end
122
128
 
129
+ desc 'test_revision MESA_DIR', 'test and submit all tests as a packaged version'
130
+ long_desc <<-LONGDESC
131
+ Run and check all test cases residing in MESA_DIR/star/test_suite. Then
132
+ report results to MesaTestHub. Specifically, runs and checks all tests
133
+ detailed in MESA_DIR/star/test_suite/do1_test_source. Also submit version
134
+ information with test instances. Submits all test instances and version data
135
+ in a single request rather than case-by-case in `test_all`.
136
+
137
+ With --diff option, assume the checksums are up-to-date (behave like
138
+ each_test_run_and_diff), off by default.
139
+
140
+ With --force option, skip confirmation of computer details, assuming values
141
+ in ~/.mesa_test.yml are correct. Only relevant if --submit option is on
142
+ (by default it is).
143
+
144
+ With --module option, select which module to search through, with the default
145
+ being "all" (search all modules in order). Example: --module=star.
146
+
147
+ With --log option, save yml file of test results in test directory and a
148
+ summary in the test suite directory. On by default. Shut off with --no-log.
149
+
150
+ With --submit option, upload results to MESATestHub. By default, this is on.
151
+ To run without submission, use --no-submit.
152
+
153
+ With --svn option, get version info (number, author, commit message) from
154
+ svn. By default, this is on. To shut off, use --no-svn. If off, no author or
155
+ commit message are saved.
156
+ LONGDESC
157
+ option :diff, type: :boolean, default: false
158
+ option :force, type: :boolean, aliases: '-f'
159
+ option :log, type: :boolean, default: true
160
+ option :module, type: :string, default: :all
161
+ option :submit, type: :boolean, default: true
162
+ option :svn, type: :boolean, default: true
163
+ def test_revision(mesa_dir)
164
+ s = create_and_check_submitter(force: options[:force]) if options[:submit]
165
+ m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
166
+ mod: options[:module].downcase.to_sym,
167
+ use_svn: options[:svn])
168
+
169
+ # run all tests
170
+ m.each_test_run_and_diff(log_results: options[:log],
171
+ mod: options[:module].downcase.to_sym)
172
+
173
+ # submit all tests
174
+ s.submit_revision(m) if options[:submit]
175
+ end
176
+
177
+
123
178
  desc 'submit_one MESA_DIR TEST_CASE', 'submit one completed test case'
124
179
  long_desc <<-LONGDESC
125
180
  Load complete test case data from existing YAML file and submit to MESA.
@@ -163,13 +218,19 @@ class MesaTest < Thor
163
218
 
164
219
  With --module option, select which module to search through, with the default
165
220
  being "all" (search all modules in order). Example: --module=star.
221
+
222
+ With --svn option, confirm version number with svn rather than trusting
223
+ self-reported version number. By default, this is on. To run without using
224
+ svn, use --no-svn.
166
225
  LONGDESC
167
226
  option :force, type: :boolean, aliases: '-f'
168
227
  option :module, type: :string, aliases: '-m', default: 'all'
228
+ option :svn, type: :boolean, default: true
169
229
  def submit_all(mesa_dir)
170
230
  s = create_and_check_submitter(force: options[:force])
171
231
  m = create_and_check_mesa(mesa_dir: mesa_dir,
172
- mod: options[:module].downcase.to_sym)
232
+ mod: options[:module].downcase.to_sym,
233
+ use_svn: options[:svn])
173
234
 
174
235
  # load all test data
175
236
  m.each_test_load_results(mod: options[:module].downcase.to_sym)
@@ -178,6 +239,42 @@ class MesaTest < Thor
178
239
  s.submit_all(m, options[:module].downcase.to_sym)
179
240
  end
180
241
 
242
+ desc 'submit_revision MESA_DIR', 'submit version with pre-run test cases'
243
+ long_desc <<-LONGDESC
244
+ Load results for all test cases residing in MESA_DIR/star/test_suite. Then
245
+ submit results to MesaTestHub. Specifically, loads and submits all tests
246
+ detailed in MESA_DIR/star/test_suite/do1_test_source along with version data
247
+ (number, author, and commit message). Unlike submit_all, this submits all
248
+ test cases AND version info in a single request rather than just the cases
249
+ in series.
250
+
251
+ With --force option, skip confirmation of computer details, assuming values
252
+ in ~/.mesa_test.yml are correct.
253
+
254
+ With --module option, select which module to search through, with the default
255
+ being "all" (search all modules in order). Example: --module=star.
256
+
257
+ With --svn option, get version info (number, author, commit message) from
258
+ svn. By default, this is on. To shut off, use --no-svn. If off, no author or
259
+ commit message are submitted.
260
+ LONGDESC
261
+ option :force, type: :boolean, aliases: '-f'
262
+ option :module, type: :string, aliases: '-m', default: 'all'
263
+ option :svn, type: :boolean, default: true
264
+ def submit_revision(mesa_dir)
265
+ s = create_and_check_submitter(force: options[:force])
266
+ m = create_and_check_mesa(mesa_dir: mesa_dir,
267
+ mod: options[:module].downcase.to_sym,
268
+ use_svn: options[:svn])
269
+
270
+ # load all test data
271
+ m.each_test_load_results(mod: options[:module].downcase.to_sym)
272
+
273
+ # submit all tests
274
+ s.submit_revision(m)
275
+ end
276
+
277
+
181
278
  desc 'install VERSION_NUMBER MESA_DIR', 'download and install mesa release '\
182
279
  'VERSION_NUMBER to directory MESA_DIR'
183
280
  long_desc <<-LONGDESC
@@ -262,6 +359,86 @@ class MesaTest < Thor
262
359
  m.destroy if successfully_submitted && options[:destroy]
263
360
  end
264
361
 
362
+ desc 'install_and_test_revision VERSION_NUMBER MESA_DIR', 'download and '\
363
+ 'install mesa release VERSION_NUMBER to directory MESA_DIR, run all tests,'\
364
+ 'and submit version and test data in single request.'
365
+ long_desc <<-LONGDESC
366
+ Calls to svn to install mesa release VERSION_NUMBER into the directory
367
+ MESA_DIR. Basically just an svn checkout followed by going into the directory
368
+ and running ./clean and ./install. SDK or compilers must be set up prior.
369
+ Once installation is complete, run the test suite, and report results
370
+ (version and all test data) to MesaTestHub in a single request (whereas
371
+ install_and_test_all only submits test data and does so in a serial fashion).
372
+ Does not affect the user's MESA_DIR or other environment variables. This is
373
+ basically a shortcut for running
374
+
375
+ `mesa_test install SOME_VERSION SOME_DIR`
376
+
377
+ followed by
378
+
379
+ `mesa_test test_revision SOME_DIR`
380
+
381
+ If VERSION_NUMBER is 'non-paxton' or 'non_paxton', then the most recent
382
+ commit by anyone other than Bill Paxton ('bill_paxton') is tested and the
383
+ --diff flag is ignored and set to false. Note that using 'HEAD' for version
384
+ number will automatically select the most recent release, regardless of the
385
+ author.
386
+
387
+ With --diff option, assume the checksums are up-to-date (behave like
388
+ each_test_run_and_diff), off by default.
389
+
390
+ Use flag --destroy to self destruct MESA_DIR after successful test
391
+ submission. Essentially does rm -rf MESA_DIR after the test suite. Useful
392
+ for automated testing without piling up disk space.
393
+
394
+ Use flag --force to skip confirmation of computer details if they can be
395
+ read from ~/.mesa_test.yml.
396
+
397
+ With --log option, save yml file of test results in test directory and a
398
+ summary in the test suite directory. On by default. Shut off with --no-log.
399
+
400
+ With --module option, select which module to search through, with the default
401
+ being "all". Example: --module=star.
402
+
403
+ With --svn option, get version info (number, author, commit message) from
404
+ svn. By default, this is on. To shut off, use --no-svn. If off, no author or
405
+ commit message are submitted.
406
+ LONGDESC
407
+ option :diff, type: :boolean, default: false
408
+ option :destroy, type: :boolean
409
+ option :log, type: :boolean, default: true
410
+ option :force, type: :boolean, aliases: '-f'
411
+ option :module, type: :string, aliases: '-m', default: 'all'
412
+ option :svn, type: :boolean, default: true
413
+ def install_and_test_all(version, mesa_dir)
414
+ s = create_and_check_submitter(force: options[:force])
415
+ mesa_version, diff = filter_version_and_diff(version, s)
416
+ if mesa_version.nil?
417
+ puts 'No commits by anyone other than Bill Paxton since '\
418
+ "r#{s.last_tested}. Exiting."
419
+ return
420
+ end
421
+ m = Mesa.download(version_number: mesa_version, new_mesa_dir: mesa_dir,
422
+ use_svn: options[:svn])
423
+ m.clean
424
+ m.install
425
+
426
+ m = create_and_check_mesa(mesa_dir: mesa_dir, diff: diff,
427
+ mod: options[:module].downcase.to_sym,
428
+ use_svn: options[:svn])
429
+
430
+ # run all tests. Don't be fooled, whether or not a diff happens is set
431
+ # by the diff attribute of the mesa object, set above.
432
+ m.each_test_run_and_diff(mod: options[:module].downcase.to_sym,
433
+ log_results: options[:log])
434
+
435
+ # submit all tests
436
+ successfully_submitted = s.submit_revision(m)
437
+
438
+ # if requested, and if submission successful, destroy the directory
439
+ m.destroy if successfully_submitted && options[:destroy]
440
+ end
441
+
265
442
  private
266
443
 
267
444
  def check_user_and_computer(submitter)
@@ -287,8 +464,8 @@ class MesaTest < Thor
287
464
  s
288
465
  end
289
466
 
290
- def create_and_check_mesa(mesa_dir:, diff: true, mod: :all)
291
- m = Mesa.new mesa_dir: mesa_dir
467
+ def create_and_check_mesa(mesa_dir:, diff: true, mod: :all, use_svn: true)
468
+ m = Mesa.new mesa_dir: mesa_dir, use_svn: use_svn
292
469
  unless m.installed?
293
470
  raise MesaDirError, "Invalid MESA_DIR: #{mesa_dir}. Please download " \
294
471
  'and install a valid MESA version or provide the path to one.'
data/lib/mesa_test.rb CHANGED
@@ -246,7 +246,11 @@ e-mail and password will be stored in plain text.'
246
246
  platform_version: platform_version,
247
247
  omp_num_threads: test_case.test_omp_num_threads,
248
248
  success_type: test_case.success_type,
249
- failure_type: test_case.failure_type
249
+ failure_type: test_case.failure_type,
250
+ steps: test_case.steps,
251
+ retries: test_case.retries,
252
+ backups: test_case.backups,
253
+ summary_text: test_case.summary_text
250
254
  }
251
255
 
252
256
  # enter in test-specific data, DISABLED FOR NOW
@@ -258,6 +262,57 @@ e-mail and password will be stored in plain text.'
258
262
  res
259
263
  end
260
264
 
265
+ def revision_submit_params(mesa)
266
+ mesa.load_svn_data if mesa.use_svn?
267
+ # version gives data about version
268
+ # user gives data about the user and computer submitting information
269
+ # instances is array of hashes that identify test instances (more below)
270
+ res = {
271
+ version: {number: mesa.version_number},
272
+ user: {email: email, password: password, computer: computer_name},
273
+ instances: []
274
+ }
275
+ if mesa.use_svn?
276
+ res[:version][:author] = mesa.svn_author
277
+ res[:version][:log] = mesa.svn_log
278
+ end
279
+
280
+ # hold on to test case names that fail in synthesizing params
281
+ has_errors = []
282
+
283
+ # each instance has basic information in :test_instance and extra
284
+ # information that requires the web app to work, stored in :extra
285
+ mesa.test_names.each do |mod, names|
286
+ names.each do |test_name|
287
+ begin
288
+ test_case = mesa.test_cases[mod][test_name]
289
+ res[:instances] << {
290
+ test_instance: {
291
+ runtime_seconds: test_case.runtime_seconds,
292
+ passed: test_case.passed?,
293
+ compiler: compiler,
294
+ compiler_version: compiler_version,
295
+ platform_version: platform_version,
296
+ omp_num_threads: test_case.test_omp_num_threads,
297
+ success_type: test_case.success_type,
298
+ failure_type: test_case.failure_type,
299
+ steps: test_case.steps,
300
+ retries: test_case.retries,
301
+ backups: test_case.backups,
302
+ summary_text: test_case.summary_text
303
+ },
304
+ extra: { test_case: test_name, mod: mod }
305
+ }
306
+ rescue TestCaseDirError
307
+ shell.say "Passage status for #{test_case.test_name} not yet "\
308
+ 'known. Run test first and then submit.', :red
309
+ has_errors.append(test_case)
310
+ end
311
+ end
312
+ end
313
+ [res, has_errors]
314
+ end
315
+
261
316
  def confirm_computer
262
317
  uri = URI.parse(base_uri + '/check_computer.json')
263
318
  https = Net::HTTP.new(uri.hostname, uri.port)
@@ -271,7 +326,6 @@ e-mail and password will be stored in plain text.'
271
326
  password: password,
272
327
  computer_name: computer_name
273
328
  }.to_json
274
-
275
329
  JSON.parse(https.request(request).body).to_hash
276
330
  end
277
331
 
@@ -351,15 +405,53 @@ e-mail and password will be stored in plain text.'
351
405
  # SUBMITTED (irrespective of passing status)
352
406
  success
353
407
  end
408
+
409
+ # similar to submit_all, but does EVERYTHING in one post, including
410
+ # version information. No support for individual modules now.
411
+ def submit_revision(mesa)
412
+ uri = URI.parse(base_uri + '/versions/submit_revision.json')
413
+ https = Net::HTTP.new(uri.hostname, uri.port)
414
+ https.use_ssl = true if base_uri.include? 'https'
415
+
416
+ request = Net::HTTP::Post.new(
417
+ uri,
418
+ initheader = { 'Content-Type' => 'application/json' }
419
+ )
420
+ request_data, error_cases = revision_submit_params(mesa)
421
+ if request_data[:instances].empty?
422
+ shell.say "No completed test data found in #{mesa.mesa_dir}. Aborting.",
423
+ :red
424
+ return false
425
+ end
426
+ request.body = request_data.to_json
427
+
428
+ # verbose = true
429
+ # puts "\n" if verbose
430
+ # puts JSON.parse(request.body).to_hash if verbose
431
+
432
+ response = https.request request
433
+ # puts JSON.parse(response.body).to_hash if verbose
434
+ if !response.is_a? Net::HTTPCreated
435
+ shell.say "\nFailed to submit some or all cases and/or version data.",
436
+ :red
437
+ false
438
+ elsif !error_cases.empty?
439
+ shell.say "\nFailed to gather data for the following cases:", :red
440
+ error_cases.each { |tc| shell.say " #{tc.test_name}", :red }
441
+ false
442
+ end
443
+ true
444
+ end
354
445
  end
355
446
 
356
447
  class Mesa
357
448
  SVN_URI = 'svn://svn.code.sf.net/p/mesa/code/trunk'.freeze
358
449
 
359
- attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell
450
+ attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell,
451
+ :svn_version, :svn_author, :svn_log
360
452
  attr_accessor :update_checksums
361
453
 
362
- def self.download(version_number: nil, new_mesa_dir: nil)
454
+ def self.download(version_number: nil, new_mesa_dir: nil, use_svn: true)
363
455
  new_mesa_dir ||= File.join(ENV['HOME'], 'mesa-test-r' + version_number.to_s)
364
456
  success = bash_execute(
365
457
  "svn co -r #{version_number} " \
@@ -369,7 +461,7 @@ class Mesa
369
461
  raise MesaDirError, 'Encountered a problem in download mesa ' \
370
462
  "revision #{version_number}."
371
463
  end
372
- Mesa.new(mesa_dir: new_mesa_dir)
464
+ Mesa.new(mesa_dir: new_mesa_dir, use_svn: use_svn)
373
465
  end
374
466
 
375
467
  def self.log_since(last_tested = DEFAULT_REVISION)
@@ -418,8 +510,9 @@ class Mesa
418
510
  nil
419
511
  end
420
512
 
421
- def initialize(mesa_dir: ENV['MESA_DIR'])
513
+ def initialize(mesa_dir: ENV['MESA_DIR'], use_svn: true)
422
514
  @mesa_dir = mesa_dir
515
+ @use_svn = use_svn
423
516
  @update_checksums = false
424
517
 
425
518
  # these get populated by calling #load_test_data
@@ -429,16 +522,39 @@ class Mesa
429
522
 
430
523
  # way to output colored text
431
524
  @shell = Thor::Shell::Color.new
525
+
526
+ # these can be populated by calling load_svn_data
527
+ @svn_version = nil
528
+ @svn_author = nil
529
+ @svn_log = nil
530
+ load_svn_data if use_svn?
531
+ end
532
+
533
+ def use_svn?
534
+ @use_svn
432
535
  end
433
536
 
434
537
  def version_number
435
- # prefer svn's reported version number
436
- version = svn_version_number
538
+ version = @svn_version || 0
437
539
  # fall back to MESA_DIR/data's version number svn didn't work
438
540
  version = data_version_number unless version > 0
439
541
  version
440
542
  end
441
543
 
544
+ def log_entry
545
+ `svn log #{SVN_URI} -r #{version_number}`
546
+ end
547
+
548
+ def load_svn_data
549
+ # if this number is bad, #version_number will use fallback method
550
+ @svn_version = svn_version_number
551
+ lines = log_entry.split("\n").reject { |line| line =~ /^-+$/ or line.empty?}
552
+ data_line = lines.shift
553
+ revision, author, date, length = data_line.split('|')
554
+ @svn_author = author.strip
555
+ @svn_log = lines.join("\n").strip
556
+ end
557
+
442
558
  # get version number from svn (preferred method)
443
559
  def svn_version_number
444
560
  return `svnversion #{mesa_dir}`.strip.to_i
@@ -728,7 +844,8 @@ end
728
844
  class MesaTestCase
729
845
  attr_reader :test_name, :mesa_dir, :mesa, :success_string, :final_model,
730
846
  :failure_msg, :success_msg, :photo, :runtime_seconds,
731
- :test_omp_num_threads, :mesa_version, :shell, :mod
847
+ :test_omp_num_threads, :mesa_version, :shell, :mod, :retries,
848
+ :backups, :steps, :runtime_minutes, :summary_text
732
849
  attr_accessor :data_names, :data_types, :failure_type, :success_type,
733
850
  :outcome
734
851
 
@@ -750,6 +867,11 @@ class MesaTestCase
750
867
  @outcome = :not_tested
751
868
  @runtime_seconds = 0
752
869
  @test_omp_num_threads = 1
870
+ @runtime_minutes = 0
871
+ @retries = 0
872
+ @backups = 0
873
+ @steps = 0
874
+ @summary_text = ''
753
875
  unless MesaTestCase.modules.include? mod
754
876
  raise TestCaseDirError, "Invalid module: #{mod}. Must be one of: " +
755
877
  MesaTestCase.modules.join(', ')
@@ -794,7 +916,7 @@ class MesaTestCase
794
916
  false
795
917
  else
796
918
  raise TestCaseDirError, 'Cannot determine pass/fail status of ' \
797
- '#{test_name} yet.'
919
+ "#{test_name} yet."
798
920
  end
799
921
  end
800
922
 
@@ -886,7 +1008,12 @@ class MesaTestCase
886
1008
  'outcome' => outcome,
887
1009
  'omp_num_threads' => test_omp_num_threads,
888
1010
  'success_type' => success_type,
889
- 'failure_type' => failure_type
1011
+ 'failure_type' => failure_type,
1012
+ 'runtime_minutes' => runtime_minutes,
1013
+ 'retries' => retries,
1014
+ 'backups' => backups,
1015
+ 'steps' => steps,
1016
+ 'summary_text' => summary_text
890
1017
  }
891
1018
  File.open(save_file, 'w') { |f| f.write(YAML.dump(res)) }
892
1019
  shell.say "Successfully saved results to file #{save_file}.\n", :green
@@ -909,6 +1036,11 @@ class MesaTestCase
909
1036
  @test_omp_num_threads = data['omp_num_threads'] || @test_omp_num_threads
910
1037
  @success_type = data['success_type'] || @success_type
911
1038
  @failure_type = data['failure_type'] || @failure_type
1039
+ @runtime_minutes = data['runtime_minutes'] || @runtime_minutes
1040
+ @retries = data['retries'] || @retries
1041
+ @backups = data['backups'] || @backups
1042
+ @steps = data['steps'] || @steps
1043
+ @summary_text = data['summary_text'] || @summary_text
912
1044
 
913
1045
  # convert select data to symbols since that is how they are used
914
1046
  @outcome = @outcome.to_sym if @outcome
@@ -918,6 +1050,31 @@ class MesaTestCase
918
1050
  shell.say "Done loading data from #{load_file}.\n", :green
919
1051
  end
920
1052
 
1053
+ def load_summary_data
1054
+ out_data = parse_out
1055
+ @runtime_minutes = out_data[:runtime_minutes]
1056
+ @retries = out_data[:retries]
1057
+ @backups = out_data[:backups]
1058
+ @steps = out_data[:steps]
1059
+ @summary_text = get_summary_text
1060
+ end
1061
+
1062
+ def parse_out
1063
+ runtime_minutes = 0
1064
+ retries = 0
1065
+ backups = 0
1066
+ steps = 0
1067
+ run_summaries.each do |summary|
1068
+ summary =~ /^\s*runtime\s*\(minutes\),\s+retries,\s+backups,\ssteps\s+(\d+\.?\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/
1069
+ runtime_minutes += $1.to_f
1070
+ retries += $2.to_i
1071
+ backups += $3.to_i
1072
+ steps += $4.to_i
1073
+ end
1074
+ {runtime_minutes: runtime_minutes, retries: retries, backups: backups,
1075
+ steps: steps}
1076
+ end
1077
+
921
1078
  private
922
1079
 
923
1080
  def data_types
@@ -1085,6 +1242,9 @@ class MesaTestCase
1085
1242
 
1086
1243
  # only check restart/photo if we get through run successfully
1087
1244
  check_restart if check_run
1245
+
1246
+ # get reported runtime, retries, backups, and steps
1247
+ load_summary_data
1088
1248
  end
1089
1249
 
1090
1250
  # append contents of err.txt to end of out.txt, then delete err.txt
@@ -1130,6 +1290,58 @@ class MesaTestCase
1130
1290
  return unless File.exist?(final_model)
1131
1291
  FileUtils.rm(final_model)
1132
1292
  end
1293
+
1294
+ # helpers for getting run summaries
1295
+ def run_summaries
1296
+ # look at all lines in out.txt
1297
+ lines = IO.readlines(File.join(test_case_dir, 'out.txt'))
1298
+
1299
+ # find lines with summary information
1300
+ summary_line_numbers = []
1301
+ lines.each_with_index do |line, i|
1302
+ if line =~ /^\s*runtime \(minutes\),\s+retries,\s+backups,\ssteps/
1303
+ summary_line_numbers << i
1304
+ end
1305
+ end
1306
+
1307
+ # find lines indicating passage or failure of runs and restarts
1308
+ run_finish_line_numbers = []
1309
+ restart_finish_line_numbers = []
1310
+ lines.each_with_index do |line, i|
1311
+ if line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+restart/
1312
+ restart_finish_line_numbers << i
1313
+ elsif line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+run/
1314
+ run_finish_line_numbers << i
1315
+ end
1316
+ end
1317
+
1318
+ # only keep summaries that correspond to runs rather than restart
1319
+ summary_line_numbers.select do |i|
1320
+ run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
1321
+ end.map { |line_number| lines[line_number] }
1322
+ end
1323
+
1324
+ def get_summary_text
1325
+ lines = IO.readlines(File.join(test_case_dir, 'out.txt')).select do |line|
1326
+ line =~ /^\s*runtime/
1327
+ end.join
1328
+ end
1329
+
1330
+ def run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
1331
+ # iterate from starting line (a summary line) up to largest PASS/FAIL
1332
+ # line, bail out if summary line is beyond any PASS/FAIL line
1333
+ max_line = run_finish_line_numbers.max || 0
1334
+ max_line = [max_line, (restart_finish_line_numbers.max || 0)].max
1335
+ return false if i > max_line
1336
+ # return true if next PASS/FAIL line is for a run and fail if it is for a
1337
+ # restart
1338
+ i.upto(max_line) do |j|
1339
+ return true if run_finish_line_numbers.include?(j)
1340
+ return false if restart_finish_line_numbers.include?(j)
1341
+ end
1342
+ false
1343
+ end
1344
+
1133
1345
  end
1134
1346
 
1135
1347
  ################################
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mesa_test
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.20
4
+ version: 0.0.21
5
5
  platform: ruby
6
6
  authors:
7
7
  - William Wolf
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-12-12 00:00:00.000000000 Z
11
+ date: 2017-01-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: json
@@ -82,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
82
82
  version: '0'
83
83
  requirements: []
84
84
  rubyforge_project:
85
- rubygems_version: 2.6.14
85
+ rubygems_version: 2.7.4
86
86
  signing_key:
87
87
  specification_version: 4
88
88
  summary: Command line tool for running and reporting the MESA test suites.