mesa_test 0.0.20 → 0.0.21
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/bin/mesa_test +181 -4
- data/lib/mesa_test.rb +223 -11
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 959837654b03c806d656305dbbf32b850a155b85e17df6c81306d96483b91573
|
4
|
+
data.tar.gz: d690529aa2e9b54fc8cee25753cf6270a3b7adabec2c0879f84b3930103df54a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0ea095803d606961091d77e13a0cd406fbec36d962d70abb853023e334fcc294c7c0cfc24ca4f166c51959094832c491622ec5a3f621348d15df0c3789adb27b
|
7
|
+
data.tar.gz: 9f313f3c236fee9cd94f51bb658d04103de95d0c6d1d9c0f2d70eb504bdbe9719025c020cfbe7868f05727bc32f1612eedd36f41f4a2426824b4817306583511
|
data/bin/mesa_test
CHANGED
@@ -101,16 +101,22 @@ class MesaTest < Thor
|
|
101
101
|
|
102
102
|
With --submit option, upload results to MESATestHub. By default, this is on.
|
103
103
|
To run without submission, use --no-submit.
|
104
|
+
|
105
|
+
With --svn option, confirm version number with svn rather than trusting
|
106
|
+
self-reported version number. By default, this is on. To run without using
|
107
|
+
svn, use --no-svn.
|
104
108
|
LONGDESC
|
105
109
|
option :diff, type: :boolean, default: false
|
106
110
|
option :force, type: :boolean, aliases: '-f'
|
107
111
|
option :log, type: :boolean, default: true
|
108
112
|
option :module, type: :string, default: :all
|
109
113
|
option :submit, type: :boolean, default: true
|
114
|
+
option :svn, type: :boolean, default: true
|
110
115
|
def test_all(mesa_dir)
|
111
116
|
s = create_and_check_submitter(force: options[:force]) if options[:submit]
|
112
117
|
m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
|
113
|
-
mod: options[:module].downcase.to_sym
|
118
|
+
mod: options[:module].downcase.to_sym,
|
119
|
+
use_svn: options[:svn])
|
114
120
|
|
115
121
|
# run all tests
|
116
122
|
m.each_test_run_and_diff(log_results: options[:log],
|
@@ -120,6 +126,55 @@ class MesaTest < Thor
|
|
120
126
|
s.submit_all(m, options[:module].downcase.to_sym) if options[:submit]
|
121
127
|
end
|
122
128
|
|
129
|
+
desc 'test_revision MESA_DIR', 'test and submit all tests as a packaged version'
|
130
|
+
long_desc <<-LONGDESC
|
131
|
+
Run and check all test cases residing in MESA_DIR/star/test_suite. Then
|
132
|
+
report results to MesaTestHub. Specifically, runs and checks all tests
|
133
|
+
detailed in MESA_DIR/star/test_suite/do1_test_source. Also submit version
|
134
|
+
information with test instances. Submits all test instances and version data
|
135
|
+
in a single request rather than case-by-case in `test_all`.
|
136
|
+
|
137
|
+
With --diff option, assume the checksums are up-to-date (behave like
|
138
|
+
each_test_run_and_diff), off by default.
|
139
|
+
|
140
|
+
With --force option, skip confirmation of computer details, assuming values
|
141
|
+
in ~/.mesa_test.yml are correct. Only relevant if --submit option is on
|
142
|
+
(by default it is).
|
143
|
+
|
144
|
+
With --module option, select which module to search through, with the default
|
145
|
+
being "all" (search all modules in order). Example: --module=star.
|
146
|
+
|
147
|
+
With --log option, save yml file of test results in test directory and a
|
148
|
+
summary in the test suite directory. On by default. Shut off with --no-log.
|
149
|
+
|
150
|
+
With --submit option, upload results to MESATestHub. By default, this is on.
|
151
|
+
To run without submission, use --no-submit.
|
152
|
+
|
153
|
+
With --svn option, get version info (number, author, commit message) from
|
154
|
+
svn. By default, this is on. To shut off, use --no-svn. If off, no author or
|
155
|
+
commit message are saved.
|
156
|
+
LONGDESC
|
157
|
+
option :diff, type: :boolean, default: false
|
158
|
+
option :force, type: :boolean, aliases: '-f'
|
159
|
+
option :log, type: :boolean, default: true
|
160
|
+
option :module, type: :string, default: :all
|
161
|
+
option :submit, type: :boolean, default: true
|
162
|
+
option :svn, type: :boolean, default: true
|
163
|
+
def test_revision(mesa_dir)
|
164
|
+
s = create_and_check_submitter(force: options[:force]) if options[:submit]
|
165
|
+
m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
|
166
|
+
mod: options[:module].downcase.to_sym,
|
167
|
+
use_svn: options[:svn])
|
168
|
+
|
169
|
+
# run all tests
|
170
|
+
m.each_test_run_and_diff(log_results: options[:log],
|
171
|
+
mod: options[:module].downcase.to_sym)
|
172
|
+
|
173
|
+
# submit all tests
|
174
|
+
s.submit_revision(m) if options[:submit]
|
175
|
+
end
|
176
|
+
|
177
|
+
|
123
178
|
desc 'submit_one MESA_DIR TEST_CASE', 'submit one completed test case'
|
124
179
|
long_desc <<-LONGDESC
|
125
180
|
Load complete test case data from existing YAML file and submit to MESA.
|
@@ -163,13 +218,19 @@ class MesaTest < Thor
|
|
163
218
|
|
164
219
|
With --module option, select which module to search through, with the default
|
165
220
|
being "all" (search all modules in order). Example: --module=star.
|
221
|
+
|
222
|
+
With --svn option, confirm version number with svn rather than trusting
|
223
|
+
self-reported version number. By default, this is on. To run without using
|
224
|
+
svn, use --no-svn.
|
166
225
|
LONGDESC
|
167
226
|
option :force, type: :boolean, aliases: '-f'
|
168
227
|
option :module, type: :string, aliases: '-m', default: 'all'
|
228
|
+
option :svn, type: :boolean, default: true
|
169
229
|
def submit_all(mesa_dir)
|
170
230
|
s = create_and_check_submitter(force: options[:force])
|
171
231
|
m = create_and_check_mesa(mesa_dir: mesa_dir,
|
172
|
-
mod: options[:module].downcase.to_sym
|
232
|
+
mod: options[:module].downcase.to_sym,
|
233
|
+
use_svn: options[:svn])
|
173
234
|
|
174
235
|
# load all test data
|
175
236
|
m.each_test_load_results(mod: options[:module].downcase.to_sym)
|
@@ -178,6 +239,42 @@ class MesaTest < Thor
|
|
178
239
|
s.submit_all(m, options[:module].downcase.to_sym)
|
179
240
|
end
|
180
241
|
|
242
|
+
desc 'submit_revision MESA_DIR', 'submit version with pre-run test cases'
|
243
|
+
long_desc <<-LONGDESC
|
244
|
+
Load results for all test cases residing in MESA_DIR/star/test_suite. Then
|
245
|
+
submit results to MesaTestHub. Specifically, loads and submits all tests
|
246
|
+
detailed in MESA_DIR/star/test_suite/do1_test_source along with version data
|
247
|
+
(number, author, and commit message). Unlike submit_all, this submits all
|
248
|
+
test cases AND version info in a single request rather than just the cases
|
249
|
+
in series.
|
250
|
+
|
251
|
+
With --force option, skip confirmation of computer details, assuming values
|
252
|
+
in ~/.mesa_test.yml are correct.
|
253
|
+
|
254
|
+
With --module option, select which module to search through, with the default
|
255
|
+
being "all" (search all modules in order). Example: --module=star.
|
256
|
+
|
257
|
+
With --svn option, get version info (number, author, commit message) from
|
258
|
+
svn. By default, this is on. To shut off, use --no-svn. If off, no author or
|
259
|
+
commit message are submitted.
|
260
|
+
LONGDESC
|
261
|
+
option :force, type: :boolean, aliases: '-f'
|
262
|
+
option :module, type: :string, aliases: '-m', default: 'all'
|
263
|
+
option :svn, type: :boolean, default: true
|
264
|
+
def submit_revision(mesa_dir)
|
265
|
+
s = create_and_check_submitter(force: options[:force])
|
266
|
+
m = create_and_check_mesa(mesa_dir: mesa_dir,
|
267
|
+
mod: options[:module].downcase.to_sym,
|
268
|
+
use_svn: options[:svn])
|
269
|
+
|
270
|
+
# load all test data
|
271
|
+
m.each_test_load_results(mod: options[:module].downcase.to_sym)
|
272
|
+
|
273
|
+
# submit all tests
|
274
|
+
s.submit_revision(m)
|
275
|
+
end
|
276
|
+
|
277
|
+
|
181
278
|
desc 'install VERSION_NUMBER MESA_DIR', 'download and install mesa release '\
|
182
279
|
'VERSION_NUMBER to directory MESA_DIR'
|
183
280
|
long_desc <<-LONGDESC
|
@@ -262,6 +359,86 @@ class MesaTest < Thor
|
|
262
359
|
m.destroy if successfully_submitted && options[:destroy]
|
263
360
|
end
|
264
361
|
|
362
|
+
desc 'install_and_test_revision VERSION_NUMBER MESA_DIR', 'download and '\
|
363
|
+
'install mesa release VERSION_NUMBER to directory MESA_DIR, run all tests,'\
|
364
|
+
'and submit version and test data in single request.'
|
365
|
+
long_desc <<-LONGDESC
|
366
|
+
Calls to svn to install mesa release VERSION_NUMBER into the directory
|
367
|
+
MESA_DIR. Basically just an svn checkout followed by going into the directory
|
368
|
+
and running ./clean and ./install. SDK or compilers must be set up prior.
|
369
|
+
Once installation is complete, run the test suite, and report results
|
370
|
+
(version and all test data) to MesaTestHub in a single request (whereas
|
371
|
+
install_and_test_all only submits test data and does so in a serial fashion).
|
372
|
+
Does not affect the user's MESA_DIR or other environment variables. This is
|
373
|
+
basically a shortcut for running
|
374
|
+
|
375
|
+
`mesa_test install SOME_VERSION SOME_DIR`
|
376
|
+
|
377
|
+
followed by
|
378
|
+
|
379
|
+
`mesa_test test_revision SOME_DIR`
|
380
|
+
|
381
|
+
If VERSION_NUMBER is 'non-paxton' or 'non_paxton', then the most recent
|
382
|
+
commit by anyone other than Bill Paxton ('bill_paxton') is tested and the
|
383
|
+
--diff flag is ignored and set to false. Note that using 'HEAD' for version
|
384
|
+
number will automatically select the most recent release, regardless of the
|
385
|
+
author.
|
386
|
+
|
387
|
+
With --diff option, assume the checksums are up-to-date (behave like
|
388
|
+
each_test_run_and_diff), off by default.
|
389
|
+
|
390
|
+
Use flag --destroy to self destruct MESA_DIR after successful test
|
391
|
+
submission. Essentially does rm -rf MESA_DIR after the test suite. Useful
|
392
|
+
for automated testing without piling up disk space.
|
393
|
+
|
394
|
+
Use flag --force to skip confirmation of computer details if they can be
|
395
|
+
read from ~/.mesa_test.yml.
|
396
|
+
|
397
|
+
With --log option, save yml file of test results in test directory and a
|
398
|
+
summary in the test suite directory. On by default. Shut off with --no-log.
|
399
|
+
|
400
|
+
With --module option, select which module to search through, with the default
|
401
|
+
being "all". Example: --module=star.
|
402
|
+
|
403
|
+
With --svn option, get version info (number, author, commit message) from
|
404
|
+
svn. By default, this is on. To shut off, use --no-svn. If off, no author or
|
405
|
+
commit message are submitted.
|
406
|
+
LONGDESC
|
407
|
+
option :diff, type: :boolean, default: false
|
408
|
+
option :destroy, type: :boolean
|
409
|
+
option :log, type: :boolean, default: true
|
410
|
+
option :force, type: :boolean, aliases: '-f'
|
411
|
+
option :module, type: :string, aliases: '-m', default: 'all'
|
412
|
+
option :svn, type: :boolean, default: true
|
413
|
+
def install_and_test_all(version, mesa_dir)
|
414
|
+
s = create_and_check_submitter(force: options[:force])
|
415
|
+
mesa_version, diff = filter_version_and_diff(version, s)
|
416
|
+
if mesa_version.nil?
|
417
|
+
puts 'No commits by anyone other than Bill Paxton since '\
|
418
|
+
"r#{s.last_tested}. Exiting."
|
419
|
+
return
|
420
|
+
end
|
421
|
+
m = Mesa.download(version_number: mesa_version, new_mesa_dir: mesa_dir,
|
422
|
+
use_svn: options[:svn])
|
423
|
+
m.clean
|
424
|
+
m.install
|
425
|
+
|
426
|
+
m = create_and_check_mesa(mesa_dir: mesa_dir, diff: diff,
|
427
|
+
mod: options[:module].downcase.to_sym,
|
428
|
+
use_svn: options[:svn])
|
429
|
+
|
430
|
+
# run all tests. Don't be fooled, whether or not a diff happens is set
|
431
|
+
# by the diff attribute of the mesa object, set above.
|
432
|
+
m.each_test_run_and_diff(mod: options[:module].downcase.to_sym,
|
433
|
+
log_results: options[:log])
|
434
|
+
|
435
|
+
# submit all tests
|
436
|
+
successfully_submitted = s.submit_revision(m)
|
437
|
+
|
438
|
+
# if requested, and if submission successful, destroy the directory
|
439
|
+
m.destroy if successfully_submitted && options[:destroy]
|
440
|
+
end
|
441
|
+
|
265
442
|
private
|
266
443
|
|
267
444
|
def check_user_and_computer(submitter)
|
@@ -287,8 +464,8 @@ class MesaTest < Thor
|
|
287
464
|
s
|
288
465
|
end
|
289
466
|
|
290
|
-
def create_and_check_mesa(mesa_dir:, diff: true, mod: :all)
|
291
|
-
m = Mesa.new mesa_dir: mesa_dir
|
467
|
+
def create_and_check_mesa(mesa_dir:, diff: true, mod: :all, use_svn: true)
|
468
|
+
m = Mesa.new mesa_dir: mesa_dir, use_svn: use_svn
|
292
469
|
unless m.installed?
|
293
470
|
raise MesaDirError, "Invalid MESA_DIR: #{mesa_dir}. Please download " \
|
294
471
|
'and install a valid MESA version or provide the path to one.'
|
data/lib/mesa_test.rb
CHANGED
@@ -246,7 +246,11 @@ e-mail and password will be stored in plain text.'
|
|
246
246
|
platform_version: platform_version,
|
247
247
|
omp_num_threads: test_case.test_omp_num_threads,
|
248
248
|
success_type: test_case.success_type,
|
249
|
-
failure_type: test_case.failure_type
|
249
|
+
failure_type: test_case.failure_type,
|
250
|
+
steps: test_case.steps,
|
251
|
+
retries: test_case.retries,
|
252
|
+
backups: test_case.backups,
|
253
|
+
summary_text: test_case.summary_text
|
250
254
|
}
|
251
255
|
|
252
256
|
# enter in test-specific data, DISABLED FOR NOW
|
@@ -258,6 +262,57 @@ e-mail and password will be stored in plain text.'
|
|
258
262
|
res
|
259
263
|
end
|
260
264
|
|
265
|
+
def revision_submit_params(mesa)
|
266
|
+
mesa.load_svn_data if mesa.use_svn?
|
267
|
+
# version gives data about version
|
268
|
+
# user gives data about the user and computer submitting information
|
269
|
+
# instances is array of hashes that identify test instances (more below)
|
270
|
+
res = {
|
271
|
+
version: {number: mesa.version_number},
|
272
|
+
user: {email: email, password: password, computer: computer_name},
|
273
|
+
instances: []
|
274
|
+
}
|
275
|
+
if mesa.use_svn?
|
276
|
+
res[:version][:author] = mesa.svn_author
|
277
|
+
res[:version][:log] = mesa.svn_log
|
278
|
+
end
|
279
|
+
|
280
|
+
# hold on to test case names that fail in synthesizing params
|
281
|
+
has_errors = []
|
282
|
+
|
283
|
+
# each instance has basic information in :test_instance and extra
|
284
|
+
# information that requires the web app to work, stored in :extra
|
285
|
+
mesa.test_names.each do |mod, names|
|
286
|
+
names.each do |test_name|
|
287
|
+
begin
|
288
|
+
test_case = mesa.test_cases[mod][test_name]
|
289
|
+
res[:instances] << {
|
290
|
+
test_instance: {
|
291
|
+
runtime_seconds: test_case.runtime_seconds,
|
292
|
+
passed: test_case.passed?,
|
293
|
+
compiler: compiler,
|
294
|
+
compiler_version: compiler_version,
|
295
|
+
platform_version: platform_version,
|
296
|
+
omp_num_threads: test_case.test_omp_num_threads,
|
297
|
+
success_type: test_case.success_type,
|
298
|
+
failure_type: test_case.failure_type,
|
299
|
+
steps: test_case.steps,
|
300
|
+
retries: test_case.retries,
|
301
|
+
backups: test_case.backups,
|
302
|
+
summary_text: test_case.summary_text
|
303
|
+
},
|
304
|
+
extra: { test_case: test_name, mod: mod }
|
305
|
+
}
|
306
|
+
rescue TestCaseDirError
|
307
|
+
shell.say "Passage status for #{test_case.test_name} not yet "\
|
308
|
+
'known. Run test first and then submit.', :red
|
309
|
+
has_errors.append(test_case)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
313
|
+
[res, has_errors]
|
314
|
+
end
|
315
|
+
|
261
316
|
def confirm_computer
|
262
317
|
uri = URI.parse(base_uri + '/check_computer.json')
|
263
318
|
https = Net::HTTP.new(uri.hostname, uri.port)
|
@@ -271,7 +326,6 @@ e-mail and password will be stored in plain text.'
|
|
271
326
|
password: password,
|
272
327
|
computer_name: computer_name
|
273
328
|
}.to_json
|
274
|
-
|
275
329
|
JSON.parse(https.request(request).body).to_hash
|
276
330
|
end
|
277
331
|
|
@@ -351,15 +405,53 @@ e-mail and password will be stored in plain text.'
|
|
351
405
|
# SUBMITTED (irrespective of passing status)
|
352
406
|
success
|
353
407
|
end
|
408
|
+
|
409
|
+
# similar to submit_all, but does EVERYTHING in one post, including
|
410
|
+
# version information. No support for individual modules now.
|
411
|
+
def submit_revision(mesa)
|
412
|
+
uri = URI.parse(base_uri + '/versions/submit_revision.json')
|
413
|
+
https = Net::HTTP.new(uri.hostname, uri.port)
|
414
|
+
https.use_ssl = true if base_uri.include? 'https'
|
415
|
+
|
416
|
+
request = Net::HTTP::Post.new(
|
417
|
+
uri,
|
418
|
+
initheader = { 'Content-Type' => 'application/json' }
|
419
|
+
)
|
420
|
+
request_data, error_cases = revision_submit_params(mesa)
|
421
|
+
if request_data[:instances].empty?
|
422
|
+
shell.say "No completed test data found in #{mesa.mesa_dir}. Aborting.",
|
423
|
+
:red
|
424
|
+
return false
|
425
|
+
end
|
426
|
+
request.body = request_data.to_json
|
427
|
+
|
428
|
+
# verbose = true
|
429
|
+
# puts "\n" if verbose
|
430
|
+
# puts JSON.parse(request.body).to_hash if verbose
|
431
|
+
|
432
|
+
response = https.request request
|
433
|
+
# puts JSON.parse(response.body).to_hash if verbose
|
434
|
+
if !response.is_a? Net::HTTPCreated
|
435
|
+
shell.say "\nFailed to submit some or all cases and/or version data.",
|
436
|
+
:red
|
437
|
+
false
|
438
|
+
elsif !error_cases.empty?
|
439
|
+
shell.say "\nFailed to gather data for the following cases:", :red
|
440
|
+
error_cases.each { |tc| shell.say " #{tc.test_name}", :red }
|
441
|
+
false
|
442
|
+
end
|
443
|
+
true
|
444
|
+
end
|
354
445
|
end
|
355
446
|
|
356
447
|
class Mesa
|
357
448
|
SVN_URI = 'svn://svn.code.sf.net/p/mesa/code/trunk'.freeze
|
358
449
|
|
359
|
-
attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell
|
450
|
+
attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell,
|
451
|
+
:svn_version, :svn_author, :svn_log
|
360
452
|
attr_accessor :update_checksums
|
361
453
|
|
362
|
-
def self.download(version_number: nil, new_mesa_dir: nil)
|
454
|
+
def self.download(version_number: nil, new_mesa_dir: nil, use_svn: true)
|
363
455
|
new_mesa_dir ||= File.join(ENV['HOME'], 'mesa-test-r' + version_number.to_s)
|
364
456
|
success = bash_execute(
|
365
457
|
"svn co -r #{version_number} " \
|
@@ -369,7 +461,7 @@ class Mesa
|
|
369
461
|
raise MesaDirError, 'Encountered a problem in download mesa ' \
|
370
462
|
"revision #{version_number}."
|
371
463
|
end
|
372
|
-
Mesa.new(mesa_dir: new_mesa_dir)
|
464
|
+
Mesa.new(mesa_dir: new_mesa_dir, use_svn: use_svn)
|
373
465
|
end
|
374
466
|
|
375
467
|
def self.log_since(last_tested = DEFAULT_REVISION)
|
@@ -418,8 +510,9 @@ class Mesa
|
|
418
510
|
nil
|
419
511
|
end
|
420
512
|
|
421
|
-
def initialize(mesa_dir: ENV['MESA_DIR'])
|
513
|
+
def initialize(mesa_dir: ENV['MESA_DIR'], use_svn: true)
|
422
514
|
@mesa_dir = mesa_dir
|
515
|
+
@use_svn = use_svn
|
423
516
|
@update_checksums = false
|
424
517
|
|
425
518
|
# these get populated by calling #load_test_data
|
@@ -429,16 +522,39 @@ class Mesa
|
|
429
522
|
|
430
523
|
# way to output colored text
|
431
524
|
@shell = Thor::Shell::Color.new
|
525
|
+
|
526
|
+
# these can be populated by calling load_svn_data
|
527
|
+
@svn_version = nil
|
528
|
+
@svn_author = nil
|
529
|
+
@svn_log = nil
|
530
|
+
load_svn_data if use_svn?
|
531
|
+
end
|
532
|
+
|
533
|
+
def use_svn?
|
534
|
+
@use_svn
|
432
535
|
end
|
433
536
|
|
434
537
|
def version_number
|
435
|
-
|
436
|
-
version = svn_version_number
|
538
|
+
version = @svn_version || 0
|
437
539
|
# fall back to MESA_DIR/data's version number svn didn't work
|
438
540
|
version = data_version_number unless version > 0
|
439
541
|
version
|
440
542
|
end
|
441
543
|
|
544
|
+
def log_entry
|
545
|
+
`svn log #{SVN_URI} -r #{version_number}`
|
546
|
+
end
|
547
|
+
|
548
|
+
def load_svn_data
|
549
|
+
# if this number is bad, #version_number will use fallback method
|
550
|
+
@svn_version = svn_version_number
|
551
|
+
lines = log_entry.split("\n").reject { |line| line =~ /^-+$/ or line.empty?}
|
552
|
+
data_line = lines.shift
|
553
|
+
revision, author, date, length = data_line.split('|')
|
554
|
+
@svn_author = author.strip
|
555
|
+
@svn_log = lines.join("\n").strip
|
556
|
+
end
|
557
|
+
|
442
558
|
# get version number from svn (preferred method)
|
443
559
|
def svn_version_number
|
444
560
|
return `svnversion #{mesa_dir}`.strip.to_i
|
@@ -728,7 +844,8 @@ end
|
|
728
844
|
class MesaTestCase
|
729
845
|
attr_reader :test_name, :mesa_dir, :mesa, :success_string, :final_model,
|
730
846
|
:failure_msg, :success_msg, :photo, :runtime_seconds,
|
731
|
-
:test_omp_num_threads, :mesa_version, :shell, :mod
|
847
|
+
:test_omp_num_threads, :mesa_version, :shell, :mod, :retries,
|
848
|
+
:backups, :steps, :runtime_minutes, :summary_text
|
732
849
|
attr_accessor :data_names, :data_types, :failure_type, :success_type,
|
733
850
|
:outcome
|
734
851
|
|
@@ -750,6 +867,11 @@ class MesaTestCase
|
|
750
867
|
@outcome = :not_tested
|
751
868
|
@runtime_seconds = 0
|
752
869
|
@test_omp_num_threads = 1
|
870
|
+
@runtime_minutes = 0
|
871
|
+
@retries = 0
|
872
|
+
@backups = 0
|
873
|
+
@steps = 0
|
874
|
+
@summary_text = ''
|
753
875
|
unless MesaTestCase.modules.include? mod
|
754
876
|
raise TestCaseDirError, "Invalid module: #{mod}. Must be one of: " +
|
755
877
|
MesaTestCase.modules.join(', ')
|
@@ -794,7 +916,7 @@ class MesaTestCase
|
|
794
916
|
false
|
795
917
|
else
|
796
918
|
raise TestCaseDirError, 'Cannot determine pass/fail status of ' \
|
797
|
-
|
919
|
+
"#{test_name} yet."
|
798
920
|
end
|
799
921
|
end
|
800
922
|
|
@@ -886,7 +1008,12 @@ class MesaTestCase
|
|
886
1008
|
'outcome' => outcome,
|
887
1009
|
'omp_num_threads' => test_omp_num_threads,
|
888
1010
|
'success_type' => success_type,
|
889
|
-
'failure_type' => failure_type
|
1011
|
+
'failure_type' => failure_type,
|
1012
|
+
'runtime_minutes' => runtime_minutes,
|
1013
|
+
'retries' => retries,
|
1014
|
+
'backups' => backups,
|
1015
|
+
'steps' => steps,
|
1016
|
+
'summary_text' => summary_text
|
890
1017
|
}
|
891
1018
|
File.open(save_file, 'w') { |f| f.write(YAML.dump(res)) }
|
892
1019
|
shell.say "Successfully saved results to file #{save_file}.\n", :green
|
@@ -909,6 +1036,11 @@ class MesaTestCase
|
|
909
1036
|
@test_omp_num_threads = data['omp_num_threads'] || @test_omp_num_threads
|
910
1037
|
@success_type = data['success_type'] || @success_type
|
911
1038
|
@failure_type = data['failure_type'] || @failure_type
|
1039
|
+
@runtime_minutes = data['runtime_minutes'] || @runtime_minutes
|
1040
|
+
@retries = data['retries'] || @retries
|
1041
|
+
@backups = data['backups'] || @backups
|
1042
|
+
@steps = data['steps'] || @steps
|
1043
|
+
@summary_text = data['summary_text'] || @summary_text
|
912
1044
|
|
913
1045
|
# convert select data to symbols since that is how they are used
|
914
1046
|
@outcome = @outcome.to_sym if @outcome
|
@@ -918,6 +1050,31 @@ class MesaTestCase
|
|
918
1050
|
shell.say "Done loading data from #{load_file}.\n", :green
|
919
1051
|
end
|
920
1052
|
|
1053
|
+
def load_summary_data
|
1054
|
+
out_data = parse_out
|
1055
|
+
@runtime_minutes = out_data[:runtime_minutes]
|
1056
|
+
@retries = out_data[:retries]
|
1057
|
+
@backups = out_data[:backups]
|
1058
|
+
@steps = out_data[:steps]
|
1059
|
+
@summary_text = get_summary_text
|
1060
|
+
end
|
1061
|
+
|
1062
|
+
def parse_out
|
1063
|
+
runtime_minutes = 0
|
1064
|
+
retries = 0
|
1065
|
+
backups = 0
|
1066
|
+
steps = 0
|
1067
|
+
run_summaries.each do |summary|
|
1068
|
+
summary =~ /^\s*runtime\s*\(minutes\),\s+retries,\s+backups,\ssteps\s+(\d+\.?\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/
|
1069
|
+
runtime_minutes += $1.to_f
|
1070
|
+
retries += $2.to_i
|
1071
|
+
backups += $3.to_i
|
1072
|
+
steps += $4.to_i
|
1073
|
+
end
|
1074
|
+
{runtime_minutes: runtime_minutes, retries: retries, backups: backups,
|
1075
|
+
steps: steps}
|
1076
|
+
end
|
1077
|
+
|
921
1078
|
private
|
922
1079
|
|
923
1080
|
def data_types
|
@@ -1085,6 +1242,9 @@ class MesaTestCase
|
|
1085
1242
|
|
1086
1243
|
# only check restart/photo if we get through run successfully
|
1087
1244
|
check_restart if check_run
|
1245
|
+
|
1246
|
+
# get reported runtime, retries, backups, and steps
|
1247
|
+
load_summary_data
|
1088
1248
|
end
|
1089
1249
|
|
1090
1250
|
# append contents of err.txt to end of out.txt, then delete err.txt
|
@@ -1130,6 +1290,58 @@ class MesaTestCase
|
|
1130
1290
|
return unless File.exist?(final_model)
|
1131
1291
|
FileUtils.rm(final_model)
|
1132
1292
|
end
|
1293
|
+
|
1294
|
+
# helpers for getting run summaries
|
1295
|
+
def run_summaries
|
1296
|
+
# look at all lines in out.txt
|
1297
|
+
lines = IO.readlines(File.join(test_case_dir, 'out.txt'))
|
1298
|
+
|
1299
|
+
# find lines with summary information
|
1300
|
+
summary_line_numbers = []
|
1301
|
+
lines.each_with_index do |line, i|
|
1302
|
+
if line =~ /^\s*runtime \(minutes\),\s+retries,\s+backups,\ssteps/
|
1303
|
+
summary_line_numbers << i
|
1304
|
+
end
|
1305
|
+
end
|
1306
|
+
|
1307
|
+
# find lines indicating passage or failure of runs and restarts
|
1308
|
+
run_finish_line_numbers = []
|
1309
|
+
restart_finish_line_numbers = []
|
1310
|
+
lines.each_with_index do |line, i|
|
1311
|
+
if line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+restart/
|
1312
|
+
restart_finish_line_numbers << i
|
1313
|
+
elsif line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+run/
|
1314
|
+
run_finish_line_numbers << i
|
1315
|
+
end
|
1316
|
+
end
|
1317
|
+
|
1318
|
+
# only keep summaries that correspond to runs rather than restart
|
1319
|
+
summary_line_numbers.select do |i|
|
1320
|
+
run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
|
1321
|
+
end.map { |line_number| lines[line_number] }
|
1322
|
+
end
|
1323
|
+
|
1324
|
+
def get_summary_text
|
1325
|
+
lines = IO.readlines(File.join(test_case_dir, 'out.txt')).select do |line|
|
1326
|
+
line =~ /^\s*runtime/
|
1327
|
+
end.join
|
1328
|
+
end
|
1329
|
+
|
1330
|
+
def run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
|
1331
|
+
# iterate from starting line (a summary line) up to largest PASS/FAIL
|
1332
|
+
# line, bail out if summary line is beyond any PASS/FAIL line
|
1333
|
+
max_line = run_finish_line_numbers.max || 0
|
1334
|
+
max_line = [max_line, (restart_finish_line_numbers.max || 0)].max
|
1335
|
+
return false if i > max_line
|
1336
|
+
# return true if next PASS/FAIL line is for a run and fail if it is for a
|
1337
|
+
# restart
|
1338
|
+
i.upto(max_line) do |j|
|
1339
|
+
return true if run_finish_line_numbers.include?(j)
|
1340
|
+
return false if restart_finish_line_numbers.include?(j)
|
1341
|
+
end
|
1342
|
+
false
|
1343
|
+
end
|
1344
|
+
|
1133
1345
|
end
|
1134
1346
|
|
1135
1347
|
################################
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mesa_test
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- William Wolf
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|
@@ -82,7 +82,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
82
82
|
version: '0'
|
83
83
|
requirements: []
|
84
84
|
rubyforge_project:
|
85
|
-
rubygems_version: 2.
|
85
|
+
rubygems_version: 2.7.4
|
86
86
|
signing_key:
|
87
87
|
specification_version: 4
|
88
88
|
summary: Command line tool for running and reporting the MESA test suites.
|