mesa_test 0.2.14 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/mesa_test +262 -432
- data/lib/mesa_test.rb +444 -1167
- metadata +6 -7
data/lib/mesa_test.rb
CHANGED
@@ -11,9 +11,10 @@ require 'json'
|
|
11
11
|
MesaDirError = Class.new(StandardError)
|
12
12
|
TestCaseDirError = Class.new(StandardError)
|
13
13
|
InvalidDataType = Class.new(StandardError)
|
14
|
+
GitHubError = Class.new(StandardError)
|
14
15
|
|
15
|
-
|
16
|
-
|
16
|
+
GITHUB_HTTPS = 'https://github.com/MESAHub/mesa.git'.freeze
|
17
|
+
GITHUB_SSH = 'git@github.com:MESAHub/mesa.git'.freeze
|
17
18
|
|
18
19
|
class MesaTestSubmitter
|
19
20
|
DEFAULT_URI = 'https://mesa-test-hub.herokuapp.com'.freeze
|
@@ -28,7 +29,8 @@ shown in parentheses at the end of a prompt. Pressing enter will accept the
|
|
28
29
|
default values.
|
29
30
|
|
30
31
|
To submit to MESATestHub, a valid computer name, email address, and password
|
31
|
-
are all required.
|
32
|
+
are all required. To actually run a test, you need to specify a location for
|
33
|
+
your base MESA git repository. All other data are useful, but optional. Any data
|
32
34
|
transferred to MESATestHub will be encrypted via HTTPS, but be warned that your
|
33
35
|
e-mail and password will be stored in plain text.'
|
34
36
|
# Get computer name
|
@@ -36,11 +38,6 @@ e-mail and password will be stored in plain text.'
|
|
36
38
|
"(#{s.computer_name}):", :blue)
|
37
39
|
s.computer_name = response unless response.empty?
|
38
40
|
|
39
|
-
# Get user name
|
40
|
-
response = shell.ask 'What is the name of the operator of this ' \
|
41
|
-
"computer? (#{s.user_name}):", :blue
|
42
|
-
s.user_name = response unless response.empty?
|
43
|
-
|
44
41
|
# Get user e-mail
|
45
42
|
response = shell.ask 'What is the email you can be reached ' \
|
46
43
|
"at (required)? (#{s.email}):", :blue
|
@@ -51,59 +48,54 @@ e-mail and password will be stored in plain text.'
|
|
51
48
|
"#{s.email} (required)? (#{s.password})", :blue
|
52
49
|
s.password = response unless response.empty?
|
53
50
|
|
51
|
+
# Determine if we'll use ssh or https to access github
|
52
|
+
response = shell.ask 'When accessing GitHub, which protocol do you '\
|
53
|
+
'want to use? ', :blue, limited_to: %w[ssh https]
|
54
|
+
s.github_protocol = response.strip.downcase.to_sym
|
55
|
+
|
56
|
+
# Get location of source MESA repo (the mirror)
|
57
|
+
response = shell.ask "Where is/should your mirrored MESA repository " \
|
58
|
+
"located? This is where a mirror will be stored from which test " \
|
59
|
+
"repos will be generated. You won't touch this in regular operation. " \
|
60
|
+
"(#{s.mesa_mirror}):", :blue
|
61
|
+
s.mesa_mirror = response unless response.empty?
|
62
|
+
|
63
|
+
# Get location of source MESA work (where testing happens)
|
64
|
+
response = shell.ask "Where is/should your working directory for "\
|
65
|
+
"testing be located? This is where testing actually occurs, but all "\
|
66
|
+
"files it uses are cached in the mirror repo to save time later. " \
|
67
|
+
"(#{s.mesa_work}):", :blue
|
68
|
+
s.mesa_work = response unless response.empty?
|
69
|
+
|
54
70
|
# Get platform information
|
55
71
|
response = shell.ask 'What is the platform of this computer (eg. ' \
|
56
72
|
"macOS, Ubuntu)? (#{s.platform}):", :blue
|
57
73
|
s.platform = response unless response.empty?
|
58
|
-
response = shell.ask 'What is the version of the platform (eg. 10.
|
59
|
-
"16.04)? (#{s.platform_version}):", :blue
|
74
|
+
response = shell.ask 'What is the version of the platform (eg. 10.15.5, ' \
|
75
|
+
"Ubuntu 16.04)? (#{s.platform_version}):", :blue
|
60
76
|
s.platform_version = response unless response.empty?
|
61
77
|
|
62
|
-
#
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
#
|
68
|
-
response = shell.ask 'How much RAM (in integer GB) does this computer ' \
|
69
|
-
"have (eg. 8)? (#{s.ram_gb}) ", :blue
|
70
|
-
s.ram_gb = response.to_i unless response.empty?
|
71
|
-
|
72
|
-
# Get compiler information
|
73
|
-
response = shell.ask "Which compiler are you using? (#{s.compiler}):",
|
74
|
-
:blue, limited_to: ['', 'SDK', 'gfortran', 'ifort']
|
75
|
-
s.compiler = response unless response.empty?
|
76
|
-
|
77
|
-
# Get compiler version
|
78
|
-
response = shell.ask 'What version of the compiler (eg. 20170921 or ' \
|
79
|
-
"7.2.0)? (#{s.compiler_version}): ", :blue
|
80
|
-
s.compiler_version = response unless response.empty?
|
81
|
-
|
82
|
-
# Get earliest revision to check
|
83
|
-
response = shell.ask "What's the earliest revision to search back to " \
|
84
|
-
'when finding the latest testable revision (eg. 10000)? ' \
|
85
|
-
"(#{s.last_tested}): ", :blue
|
86
|
-
s.last_tested = response.to_i unless response.empty?
|
87
|
-
|
88
|
-
# Confirm save location
|
89
|
-
response = shell.ask "This will be saved in #{s.config_file}. Press " \
|
90
|
-
'enter to accept or enter a new location:', :blue, path: true
|
91
|
-
s.config_file = response unless response.empty?
|
78
|
+
# we are powerless to do change the location for now, so stop asking
|
79
|
+
# about it
|
80
|
+
# # Confirm save location
|
81
|
+
# response = shell.ask "This will be saved in #{s.config_file}. Press " \
|
82
|
+
# 'enter to accept or enter a new location:', :blue, path: true
|
83
|
+
# s.config_file = response unless response.empty?
|
92
84
|
end
|
93
85
|
|
94
86
|
# Confirm data. If not confirmed, restart whole wizard.
|
95
87
|
if confirm_computer_data
|
96
88
|
save_computer_data
|
97
89
|
else
|
98
|
-
|
90
|
+
shell.say "Restarting wizard.\n"
|
99
91
|
setup
|
100
92
|
end
|
101
93
|
end
|
102
94
|
|
103
95
|
def self.new_from_config(
|
104
|
-
config_file: File.join(ENV['HOME'], '.mesa_test.yml'),
|
96
|
+
config_file: File.join(ENV['HOME'], '.mesa_test', 'config.yml'),
|
97
|
+
force_setup: false,
|
105
98
|
base_uri: DEFAULT_URI
|
106
|
-
# base_uri: 'http://localhost:3000'
|
107
99
|
)
|
108
100
|
new_submitter = new(config_file: config_file, base_uri: base_uri)
|
109
101
|
if force_setup
|
@@ -117,22 +109,27 @@ e-mail and password will be stored in plain text.'
|
|
117
109
|
end
|
118
110
|
|
119
111
|
attr_accessor :computer_name, :user_name, :email, :password, :platform,
|
120
|
-
:
|
121
|
-
:
|
112
|
+
:mesa_mirror, :mesa_work, :platform_version, :processor,
|
113
|
+
:config_file, :base_uri, :last_tested, :github_protocol
|
122
114
|
|
123
115
|
attr_reader :shell
|
124
116
|
|
125
117
|
# many defaults are set in body
|
126
118
|
def initialize(
|
127
|
-
computer_name: nil, user_name: nil, email: nil,
|
128
|
-
|
129
|
-
|
119
|
+
computer_name: nil, user_name: nil, email: nil, github_protocol: nil,
|
120
|
+
mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil,
|
121
|
+
config_file: nil, base_uri: nil, last_tested: nil
|
130
122
|
)
|
131
123
|
@computer_name = computer_name || Socket.gethostname.scan(/^[^\.]+\.?/)[0]
|
132
124
|
@computer_name.chomp!('.') if @computer_name
|
133
125
|
@user_name = user_name || (ENV['USER'] || ENV['USERNAME'])
|
134
126
|
@email = email || ''
|
135
127
|
@password = password || ''
|
128
|
+
@github_protocol = github_protocol || :ssh
|
129
|
+
@mesa_mirror = mesa_mirror ||
|
130
|
+
File.join(ENV['HOME'], '.mesa_test', 'mirror')
|
131
|
+
@mesa_work = mesa_work ||
|
132
|
+
File.join(ENV['HOME'], '.mesa_test', 'work')
|
136
133
|
@platform = platform
|
137
134
|
if @platform.nil?
|
138
135
|
@platform =
|
@@ -146,17 +143,15 @@ e-mail and password will be stored in plain text.'
|
|
146
143
|
end
|
147
144
|
@platform_version = platform_version || ''
|
148
145
|
@processor = processor || ''
|
149
|
-
@
|
150
|
-
|
151
|
-
@compiler_version = compiler_version || ''
|
152
|
-
@config_file = config_file || File.join(ENV['HOME'], '.mesa_test.yml')
|
146
|
+
@config_file = config_file || File.join(ENV['HOME'], '.mesa_test',
|
147
|
+
'config.yml')
|
153
148
|
@base_uri = base_uri
|
154
|
-
@last_tested = last_tested || DEFAULT_REVISION
|
155
149
|
|
156
150
|
# set up thor-proof way to get responses from user. Thor hijacks the
|
157
151
|
# gets command, so we have to use its built-in "ask" method, which is
|
158
152
|
# actually more useful
|
159
153
|
@shell = Thor::Shell::Color.new
|
154
|
+
|
160
155
|
yield self if block_given?
|
161
156
|
end
|
162
157
|
|
@@ -169,15 +164,13 @@ e-mail and password will be stored in plain text.'
|
|
169
164
|
puts 'Ready to submit the following data:'
|
170
165
|
puts '-------------------------------------------------------'
|
171
166
|
puts "Computer Name #{computer_name}"
|
172
|
-
puts "User Name #{user_name}"
|
173
167
|
puts "User email #{email}"
|
174
168
|
puts 'Password ***********'
|
169
|
+
puts "GitHub Protocol #{github_protocol}"
|
170
|
+
puts "MESA Mirror Location #{mesa_mirror}"
|
171
|
+
puts "MESA Work Location #{mesa_work}"
|
175
172
|
puts "Platform #{platform} #{platform_version}"
|
176
|
-
puts "
|
177
|
-
puts "RAM #{ram_gb} GB"
|
178
|
-
puts "Compiler #{compiler} #{compiler_version}"
|
179
|
-
puts "Last tested revision #{last_tested}"
|
180
|
-
puts "Config location #{config_file}"
|
173
|
+
# puts "Config location #{config_file}"
|
181
174
|
puts '-------------------------------------------------------'
|
182
175
|
puts ''
|
183
176
|
response = shell.ask 'Is this correct? (y/Y = Yes, anything else = No):'
|
@@ -189,158 +182,114 @@ e-mail and password will be stored in plain text.'
|
|
189
182
|
# change platforms (i.e. switch from mac to linux, or change between linux
|
190
183
|
# flavors), you should create a new computer account. Similarly, create new
|
191
184
|
# computer accounts if you change your RAM or processor. You do not need
|
192
|
-
# to change computers if you upgrade your platform (macOS 10.12 -> 10.13
|
193
|
-
# if you try different compilers
|
194
|
-
#
|
195
|
-
# Note this is NOT checked! The server really only uses the test-by-test
|
196
|
-
# quantities (platform version, compiler, compiler version) and the
|
197
|
-
# computer name. Once the computer is found (by the name) all the other
|
198
|
-
# data is assumed to be fixed. The others... probably shouldn't be here,
|
199
|
-
# but remain so you can confirm that the computer on the web server is the
|
200
|
-
# same one you think you are working with locally.
|
185
|
+
# to change computers if you upgrade your platform (macOS 10.12 -> 10.13
|
201
186
|
def save_computer_data
|
202
187
|
data_hash = {
|
203
188
|
'computer_name' => computer_name,
|
204
|
-
'user_name' => user_name,
|
205
189
|
'email' => email,
|
206
190
|
'password' => password,
|
191
|
+
'github_protocol' => github_protocol,
|
192
|
+
'mesa_mirror' => mesa_mirror,
|
193
|
+
'mesa_work' => mesa_work,
|
207
194
|
'platform' => platform,
|
208
|
-
'processor' => processor,
|
209
|
-
'ram_gb' => ram_gb,
|
210
195
|
'platform_version' => platform_version,
|
211
|
-
'compiler' => compiler,
|
212
|
-
'compiler_version' => compiler_version,
|
213
|
-
'last_tested' => last_tested
|
214
196
|
}
|
197
|
+
# make sure there's a directory to write to
|
198
|
+
unless dir_or_symlink_exists? File.dirname(config_file)
|
199
|
+
FileUtils.mkdir_p File.dirname(config_file)
|
200
|
+
end
|
215
201
|
File.open(config_file, 'w') { |f| f.write(YAML.dump(data_hash)) }
|
216
202
|
end
|
217
203
|
|
218
204
|
def load_computer_data
|
219
205
|
data_hash = YAML.safe_load(File.read(config_file), [Symbol])
|
220
206
|
@computer_name = data_hash['computer_name']
|
221
|
-
@user_name = data_hash['user_name']
|
222
207
|
@email = data_hash['email']
|
223
208
|
@password = data_hash['password']
|
209
|
+
@github_protocol = data_hash['github_protocol'].to_sym
|
210
|
+
@mesa_mirror = data_hash['mesa_mirror']
|
211
|
+
@mesa_work = data_hash['mesa_work']
|
224
212
|
@platform = data_hash['platform']
|
225
|
-
@processor = data_hash['processor']
|
226
|
-
@ram_gb = data_hash['ram_gb']
|
227
213
|
@platform_version = data_hash['platform_version']
|
228
|
-
@compiler = data_hash['compiler']
|
229
|
-
@compiler_version = data_hash['compiler_version']
|
230
|
-
@last_tested = data_hash['last_tested'] || @last_tested
|
231
214
|
end
|
232
215
|
|
233
|
-
#
|
234
|
-
#
|
235
|
-
|
236
|
-
|
237
|
-
res = {
|
238
|
-
test_case: test_case.test_name,
|
239
|
-
mod: test_case.mod,
|
240
|
-
computer: computer_name,
|
216
|
+
# Parameters to be submitted in JSON format for reporting information about
|
217
|
+
# the submitting user and computer
|
218
|
+
def submitter_params
|
219
|
+
{
|
241
220
|
email: email,
|
242
221
|
password: password,
|
243
|
-
|
244
|
-
|
245
|
-
total_runtime_seconds: test_case.total_runtime_seconds,
|
246
|
-
mesa_version: test_case.mesa_version,
|
247
|
-
passed: test_case.passed? ? 1 : 0,
|
248
|
-
compiler: test_case.compiler || compiler,
|
249
|
-
compiler_version: test_case.compiler_version || compiler_version,
|
250
|
-
platform_version: platform_version,
|
251
|
-
omp_num_threads: test_case.test_omp_num_threads,
|
252
|
-
success_type: test_case.success_type,
|
253
|
-
failure_type: test_case.failure_type,
|
254
|
-
steps: test_case.steps,
|
255
|
-
retries: test_case.retries,
|
256
|
-
backups: test_case.backups,
|
257
|
-
diff: test_case.diff,
|
258
|
-
checksum: test_case.checksum,
|
259
|
-
rn_mem: test_case.rn_mem,
|
260
|
-
re_mem: test_case.re_mem,
|
261
|
-
summary_text: test_case.summary_text
|
222
|
+
computer: computer_name,
|
223
|
+
platform_version: platform_version
|
262
224
|
}
|
263
|
-
|
264
|
-
# enter in test-specific data, DISABLED FOR NOW
|
265
|
-
# test_case.data_names.each do |data_name|
|
266
|
-
# unless test_case.data[data_name].nil?
|
267
|
-
# res[data_name] = test_case.data[data_name]
|
268
|
-
# end
|
269
|
-
# end
|
270
|
-
res
|
271
225
|
end
|
272
226
|
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
227
|
+
# Parameters to be submitted in JSON format for reporting information about
|
228
|
+
# the overall commit being tested; used even if only submitting an entire
|
229
|
+
# test. This also determines if the submission is for an entire commit
|
230
|
+
# (compilation information and every test), an empty commit (just
|
231
|
+
# compilation information), or a non-empty, but also non-entire submission
|
232
|
+
# (results for a single test without compilation information)
|
233
|
+
def commit_params(mesa, entire: true, empty: false)
|
234
|
+
# the compiler data should be able to be used as-is, but right now the
|
235
|
+
# names don't match with what the database expects, so we do some renaming
|
236
|
+
# shenanigans.
|
237
|
+
#
|
238
|
+
####################################
|
239
|
+
# THIS SHOULD GO BEFORE PRODUCTION #
|
240
|
+
{
|
241
|
+
sha: mesa.sha,
|
242
|
+
compiled: mesa.installed?,
|
243
|
+
entire: entire,
|
244
|
+
empty: empty,
|
245
|
+
}.merge(mesa.compiler_hash)
|
246
|
+
end
|
247
|
+
|
248
|
+
# Given a valid +Mesa+ object, create an array of hashes that describe the
|
249
|
+
# test cases and the test results. These will be encoded as an array of
|
250
|
+
# JSON objects.
|
251
|
+
def instance_params(mesa)
|
296
252
|
has_errors = []
|
297
|
-
|
298
|
-
|
299
|
-
# information that requires the web app to work, stored in :extra
|
300
|
-
mesa.test_names.each do |mod, names|
|
253
|
+
res = []
|
254
|
+
mesa.test_case_names.each do |mod, names|
|
301
255
|
names.each do |test_name|
|
302
256
|
begin
|
303
257
|
test_case = mesa.test_cases[mod][test_name]
|
304
|
-
res
|
305
|
-
test_instance: {
|
306
|
-
runtime_seconds: test_case.runtime_seconds,
|
307
|
-
re_time: test_case.re_time,
|
308
|
-
total_runtime_seconds: test_case.total_runtime_seconds,
|
309
|
-
passed: test_case.passed?,
|
310
|
-
compiler: test_case.compiler || compiler,
|
311
|
-
compiler_version: test_case.compiler_version || compiler_version,
|
312
|
-
platform_version: platform_version,
|
313
|
-
omp_num_threads: test_case.test_omp_num_threads,
|
314
|
-
success_type: test_case.success_type,
|
315
|
-
failure_type: test_case.failure_type,
|
316
|
-
steps: test_case.steps,
|
317
|
-
retries: test_case.retries,
|
318
|
-
backups: test_case.backups,
|
319
|
-
diff: test_case.diff,
|
320
|
-
checksum: test_case.checksum,
|
321
|
-
rn_mem: test_case.rn_mem,
|
322
|
-
re_mem: test_case.re_mem,
|
323
|
-
summary_text: test_case.summary_text
|
324
|
-
},
|
325
|
-
extra: { test_case: test_name, mod: mod }
|
326
|
-
}
|
258
|
+
res << test_case.results_hash
|
327
259
|
rescue TestCaseDirError
|
328
|
-
shell.say "
|
329
|
-
|
260
|
+
# shell.say "It appears that #{test_case.test_name} has not been "\
|
261
|
+
# 'run yet. Unable to submit data for this test.', :red
|
330
262
|
has_errors << test_case
|
331
263
|
end
|
332
264
|
end
|
333
265
|
end
|
334
|
-
|
266
|
+
unless has_errors.empty?
|
267
|
+
shell.say "The following test cases could NOT be read for submission:",
|
268
|
+
:red
|
269
|
+
has_errors.each do |test_case|
|
270
|
+
shell.say "- #{test_case.test_name}", :red
|
271
|
+
end
|
272
|
+
end
|
273
|
+
res
|
274
|
+
end
|
275
|
+
|
276
|
+
# Parameters for a single test case. +mesa+ is an instance of +Mesa+, and
|
277
|
+
# +test_case+ is an instance of MesaTestCase representing the test case to
|
278
|
+
# be submitted
|
279
|
+
def single_instance_params(test_case)
|
280
|
+
[test_case.results_hash]
|
335
281
|
end
|
336
282
|
|
283
|
+
# Phone home to testhub and confirm that computer and user are valid. Useful
|
284
|
+
# for confirming that submissions will be accepted before wasting time on a
|
285
|
+
# test later.
|
337
286
|
def confirm_computer
|
338
287
|
uri = URI.parse(base_uri + '/check_computer.json')
|
339
288
|
https = Net::HTTP.new(uri.hostname, uri.port)
|
340
289
|
https.use_ssl = base_uri.include? 'https'
|
341
290
|
|
342
291
|
request = Net::HTTP::Post.new(
|
343
|
-
uri, initheader = { 'Content-Type' => 'application/json' }
|
292
|
+
uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
|
344
293
|
)
|
345
294
|
request.body = {
|
346
295
|
email: email,
|
@@ -350,295 +299,232 @@ e-mail and password will be stored in plain text.'
|
|
350
299
|
JSON.parse(https.request(request).body).to_hash
|
351
300
|
end
|
352
301
|
|
353
|
-
#
|
354
|
-
#
|
355
|
-
|
356
|
-
|
357
|
-
def submit(test_case)
|
358
|
-
uri = URI.parse(base_uri + '/test_instances/submit.json')
|
302
|
+
# submit entire commit's worth of test cases, OR submit compilation status
|
303
|
+
# and NO test cases
|
304
|
+
def submit_commit(mesa, empty: false)
|
305
|
+
uri = URI.parse(base_uri + '/submissions/create.json')
|
359
306
|
https = Net::HTTP.new(uri.hostname, uri.port)
|
360
307
|
https.use_ssl = true if base_uri.include? 'https'
|
361
308
|
|
362
309
|
request = Net::HTTP::Post.new(
|
363
310
|
uri,
|
364
|
-
initheader = { 'Content-Type' => 'application/json' }
|
311
|
+
initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
|
365
312
|
)
|
366
|
-
begin
|
367
|
-
request.body = submit_params(test_case).to_json
|
368
|
-
rescue TestCaseDirError
|
369
|
-
shell.say "\nPassage status for #{test_case.test_name} not yet known. " \
|
370
|
-
'Run test first and then submit.', :red
|
371
|
-
return false
|
372
|
-
end
|
373
313
|
|
374
|
-
#
|
375
|
-
#
|
376
|
-
#
|
314
|
+
# create the request body for submission to the submissions API
|
315
|
+
#
|
316
|
+
# if we have an empty submission, then it is necessarily not entire.
|
317
|
+
# Similarly, a non-empty submission is necessarily entire (otherwise one
|
318
|
+
# would use +submit_instance+)
|
319
|
+
request_data = {submitter: submitter_params,
|
320
|
+
commit: commit_params(mesa, empty: empty, entire: !empty)}
|
321
|
+
# don't need test instances if it's an empty submission or if compilation
|
322
|
+
# failed
|
323
|
+
if !empty && request_data[:commit][:compiled]
|
324
|
+
request_data[:instances] = instance_params(mesa)
|
325
|
+
end
|
326
|
+
request.body = request_data.to_json
|
377
327
|
|
328
|
+
# actually do the submission
|
378
329
|
response = https.request request
|
379
|
-
# puts JSON.parse(response.body).to_hash if verbose
|
380
|
-
response.is_a? Net::HTTPCreated
|
381
|
-
end
|
382
330
|
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
success = true
|
388
|
-
mesa.test_names.each_key do |this_mod|
|
389
|
-
success &&= submit_all(mesa, mod = this_mod)
|
390
|
-
end
|
331
|
+
if !response.is_a? Net::HTTPCreated
|
332
|
+
shell.say "\nFailed to submit some or all test case instances and/or "\
|
333
|
+
'commit data.', :red
|
334
|
+
false
|
391
335
|
else
|
392
|
-
mesa.
|
393
|
-
|
394
|
-
test_case = mesa.test_cases[mod][test_name]
|
395
|
-
# try to submit and note if it does or doesn't successfully submit
|
396
|
-
submitted = false
|
397
|
-
submitted = submit(test_case) unless test_case.outcome == :not_tested
|
398
|
-
if submitted
|
399
|
-
submitted_cases << test_name
|
400
|
-
else
|
401
|
-
unsubmitted_cases << test_name
|
402
|
-
end
|
403
|
-
end
|
404
|
-
puts "\nSubmission results for #{mod} module:"
|
405
|
-
puts '#####################################'
|
406
|
-
if !submitted_cases.empty?
|
407
|
-
shell.say 'Submitted the following cases:', :green
|
408
|
-
puts submitted_cases.join("\n")
|
409
|
-
else
|
410
|
-
shell.say 'Did not successfully submit any cases.', :red
|
411
|
-
end
|
412
|
-
unless unsubmitted_cases.empty?
|
413
|
-
puts "\n\n\n"
|
414
|
-
shell.say 'Failed to submit the following cases:', :red
|
415
|
-
puts unsubmitted_cases.join("\n")
|
416
|
-
end
|
417
|
-
# return true and update last tested if all cases were submitted
|
418
|
-
success = submitted_cases.length == mesa.test_names[mod].length
|
419
|
-
if success
|
420
|
-
@last_tested = mesa.version_number
|
421
|
-
shell.say "\n\nUpdating last tested revision to #{last_tested}."
|
422
|
-
save_computer_data
|
423
|
-
end
|
336
|
+
shell.say "\nSuccessfully submitted commit #{mesa.sha}.", :green
|
337
|
+
true
|
424
338
|
end
|
425
|
-
# return boolean indicating whether or not all cases successfully
|
426
|
-
# SUBMITTED (irrespective of passing status)
|
427
|
-
success
|
428
339
|
end
|
429
340
|
|
430
|
-
#
|
431
|
-
#
|
432
|
-
def
|
433
|
-
uri = URI.parse(base_uri + '/
|
341
|
+
# submit results for a single test case instance. Does *not* report overall
|
342
|
+
# compilation status to testhub. Use an empty commit submission for that
|
343
|
+
def submit_instance(mesa, test_case)
|
344
|
+
uri = URI.parse(base_uri + '/submissions/create.json')
|
434
345
|
https = Net::HTTP.new(uri.hostname, uri.port)
|
435
346
|
https.use_ssl = true if base_uri.include? 'https'
|
436
347
|
|
437
348
|
request = Net::HTTP::Post.new(
|
438
349
|
uri,
|
439
|
-
initheader = { 'Content-Type' => 'application/json' }
|
350
|
+
initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
|
440
351
|
)
|
441
|
-
request_data, error_cases = revision_submit_params(mesa)
|
442
|
-
if request_data[:instances].empty? && mesa.installed?
|
443
|
-
shell.say "No completed test data found in #{mesa.mesa_dir}. Aborting.",
|
444
|
-
:red
|
445
|
-
return false
|
446
|
-
end
|
447
|
-
request.body = request_data.to_json
|
448
352
|
|
449
|
-
#
|
450
|
-
#
|
451
|
-
#
|
353
|
+
# create the request body for submission to the submissions API
|
354
|
+
#
|
355
|
+
# submission is not empty (there is one test case), and it is also not
|
356
|
+
# entire (... there is only test case)
|
357
|
+
request_data = {submitter: submitter_params,
|
358
|
+
commit: commit_params(mesa, empty: false, entire: false),
|
359
|
+
instances: single_instance_params(test_case)}
|
360
|
+
request.body = request_data.to_json
|
452
361
|
|
362
|
+
# actually do the submission
|
453
363
|
response = https.request request
|
454
|
-
|
364
|
+
|
455
365
|
if !response.is_a? Net::HTTPCreated
|
456
|
-
shell.say "\nFailed to submit
|
457
|
-
:red
|
458
|
-
false
|
459
|
-
elsif !error_cases.empty?
|
460
|
-
shell.say "\nFailed to gather data for the following cases:", :red
|
461
|
-
error_cases.each { |tc| shell.say " #{tc.test_name}", :red }
|
366
|
+
shell.say "\nFailed to submit #{test_case.test_name} for commit "\
|
367
|
+
"#{mesa.sha}", :red
|
462
368
|
false
|
463
369
|
else
|
464
|
-
shell.say "\nSuccessfully submitted
|
465
|
-
|
466
|
-
|
467
|
-
save_computer_data
|
468
|
-
true
|
370
|
+
shell.say "\nSuccessfully submitted instance of #{test_case.test_name} "\
|
371
|
+
"for commit #{mesa.sha}.", :green
|
372
|
+
true
|
469
373
|
end
|
470
374
|
end
|
471
375
|
end
|
472
376
|
|
473
377
|
class Mesa
|
474
|
-
|
475
|
-
|
476
|
-
attr_reader :mesa_dir, :test_data, :test_names, :test_cases, :shell,
|
477
|
-
:svn_version, :svn_author, :svn_log, :using_sdk
|
478
|
-
attr_accessor :update_checksums
|
479
|
-
|
480
|
-
def self.download(version_number: nil, new_mesa_dir: nil, use_svn: true,
|
481
|
-
using_sdk: true)
|
482
|
-
new_mesa_dir ||= File.join(ENV['HOME'], 'mesa-test-r' + version_number.to_s)
|
483
|
-
svn_command = "svn co -r #{version_number} #{SVN_URI} #{new_mesa_dir}"
|
484
|
-
success = bash_execute(svn_command)
|
485
|
-
unless success
|
486
|
-
raise MesaDirError, 'Encountered a problem in downloading mesa ' \
|
487
|
-
"revision #{version_number}. Perhaps svn isn't " \
|
488
|
-
'working properly?' + "\n\n"\
|
489
|
-
'Tried the following command: ' + svn_command
|
490
|
-
|
491
|
-
end
|
492
|
-
Mesa.new(mesa_dir: new_mesa_dir, use_svn: use_svn, using_sdk: using_sdk)
|
493
|
-
end
|
494
|
-
|
495
|
-
def self.log_since(last_tested = DEFAULT_REVISION)
|
496
|
-
# svn commit log back to, but excluding, the last revision tested
|
497
|
-
`svn log #{SVN_URI} -r #{last_tested + 1}:HEAD`
|
498
|
-
end
|
499
|
-
|
500
|
-
def self.log_lines_since(last_tested = DEFAULT_REVISION)
|
501
|
-
log_since(last_tested).split("\n").reject(&:empty?)
|
502
|
-
end
|
503
|
-
|
504
|
-
def self.add_commit(commits, revision, author)
|
505
|
-
commits << Commit.new
|
506
|
-
commits.last.revision = revision.to_i
|
507
|
-
commits.last.author = author
|
508
|
-
commits.last.message = []
|
509
|
-
end
|
510
|
-
|
511
|
-
def self.process_line(commits, line)
|
512
|
-
last = commits.last
|
513
|
-
if line =~ /^-+$/
|
514
|
-
# dashed lines separate commits
|
515
|
-
# Done with last commit (if it exists), so clean up message
|
516
|
-
last.message = last.message.join("\n") unless last.nil?
|
517
|
-
elsif line =~ /^r(\d+) \| (\w+) \| .* \| \d+ lines?$/
|
518
|
-
# first line of a commit, scrape data and make new commit
|
519
|
-
add_commit(commits, $1, $2)
|
520
|
-
else
|
521
|
-
# add lines to the message (will concatenate later to single String)
|
522
|
-
last.message << line.strip
|
523
|
-
end
|
524
|
-
end
|
525
|
-
|
526
|
-
# all commits since the given version number
|
527
|
-
def self.commits_since(last_tested = DEFAULT_REVISION)
|
528
|
-
commits = []
|
529
|
-
log_lines_since(last_tested).each { |line| process_line(commits, line) }
|
530
|
-
commits.sort_by(&:revision).reverse
|
531
|
-
end
|
378
|
+
attr_reader :mesa_dir, :mirror_dir, :names_to_numbers, :shell,
|
379
|
+
:test_case_names, :test_cases, :github_protocol
|
532
380
|
|
533
|
-
def self.
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
381
|
+
def self.checkout(sha: nil, work_dir: nil, mirror_dir: nil,
|
382
|
+
github_protocol: :ssh)
|
383
|
+
m = Mesa.new(mesa_dir: work_dir, mirror_dir: mirror_dir,
|
384
|
+
github_protocol: github_protocol)
|
385
|
+
m.checkout(new_sha: sha)
|
386
|
+
m
|
539
387
|
end
|
540
388
|
|
541
|
-
def initialize(mesa_dir: ENV['MESA_DIR'],
|
389
|
+
def initialize(mesa_dir: ENV['MESA_DIR'], mirror_dir: nil,
|
390
|
+
github_protocol: :ssh)
|
542
391
|
# absolute_path ensures that it doesn't matter where commands are executed
|
543
392
|
# from
|
544
393
|
@mesa_dir = File.absolute_path(mesa_dir)
|
545
|
-
@
|
546
|
-
|
547
|
-
|
394
|
+
@mirror_dir = File.absolute_path(mirror_dir)
|
395
|
+
|
396
|
+
# don't worry about validity of github protocol until it is needed in a
|
397
|
+
# checkout. This way you can have garbage in there if you never really need
|
398
|
+
# it.
|
399
|
+
@github_protocol = if github_protocol.respond_to? :to_sym
|
400
|
+
github_protocol.to_sym
|
401
|
+
else
|
402
|
+
github_protocol
|
403
|
+
end
|
548
404
|
|
549
405
|
# these get populated by calling #load_test_data
|
550
|
-
@test_data = {}
|
551
|
-
@test_names = {}
|
552
406
|
@test_cases = {}
|
407
|
+
@test_case_names = {}
|
408
|
+
@names_to_numbers = {}
|
553
409
|
|
554
410
|
# way to output colored text
|
555
411
|
@shell = Thor::Shell::Color.new
|
556
|
-
|
557
|
-
# these can be populated by calling load_svn_data
|
558
|
-
@svn_version = nil
|
559
|
-
@svn_author = nil
|
560
|
-
@svn_log = nil
|
561
|
-
load_svn_data if use_svn?
|
562
412
|
end
|
563
413
|
|
564
|
-
def
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
# diffs or true (DO do diffs). Only works if svn data has ALREADY been
|
571
|
-
# loaded
|
572
|
-
|
573
|
-
# don't do anything to @update_checksums if we haven't loaded svn data
|
574
|
-
return unless @svn_log
|
575
|
-
|
576
|
-
# by default, DON'T do diffs
|
577
|
-
@update_checksums = true
|
578
|
-
|
579
|
-
# list of phrases, which, if present in the log entry, will trigger diffs
|
580
|
-
[
|
581
|
-
/updated? checksums?/i,
|
582
|
-
/checksums? updated?/i,
|
583
|
-
/ready for diffs?/i,
|
584
|
-
].each { |trigger| @update_checksums = false if trigger =~ @svn_log }
|
585
|
-
if @update_checksums
|
586
|
-
shell.say "\nFrom svn log, didn't decide to tak diffs."
|
414
|
+
def checkout(new_sha: 'HEAD')
|
415
|
+
# before anything confirm that git-lfs has been installed
|
416
|
+
shell.say "\nEnsuring that git-lfs is installed... ", :blue
|
417
|
+
command = 'git lfs help >> /dev/null 2>&1'
|
418
|
+
if bash_execute(command)
|
419
|
+
shell.say "yes", :green
|
587
420
|
else
|
588
|
-
shell.say "
|
421
|
+
shell.say "no", :red
|
422
|
+
raise(GitHubError, "The command #{command} returned with an error "\
|
423
|
+
'status, indicating that git-lfs is not installed. '\
|
424
|
+
'Make sure it is installed and try again.')
|
425
|
+
end
|
426
|
+
|
427
|
+
# set up mirror if it doesn't exist
|
428
|
+
unless dir_or_symlink_exists?(mirror_dir)
|
429
|
+
shell.say "\nCreating initial mirror at #{mirror_dir}. "\
|
430
|
+
'This might take awhile...', :blue
|
431
|
+
FileUtils.mkdir_p mirror_dir
|
432
|
+
case github_protocol
|
433
|
+
when :ssh
|
434
|
+
command = "git clone --mirror #{GITHUB_SSH} #{mirror_dir}"
|
435
|
+
shell.say command
|
436
|
+
# fail loudly if this doesn't work
|
437
|
+
unless bash_execute(command)
|
438
|
+
# nuke the mirror directory since it is probably bogus (make this
|
439
|
+
# code fire off the next time checkout is done)
|
440
|
+
shell.say "Failed. Removing the [potentially corrupted] mirror.", :red
|
441
|
+
command = "rm -rf #{mirror_dir}"
|
442
|
+
shell.say command
|
443
|
+
bash_execute(command)
|
444
|
+
|
445
|
+
raise(GitHubError, 'Error while executing the following command:'\
|
446
|
+
"#{command}. Perhaps you haven't set up "\
|
447
|
+
'ssh keys with your GitHub account?')
|
448
|
+
end
|
449
|
+
when :https
|
450
|
+
command = "git clone --mirror #{GITHUB_HTTPS} #{mirror_dir}"
|
451
|
+
shell.say command
|
452
|
+
# fail loudly if this doesn't work
|
453
|
+
unless bash_execute(command)
|
454
|
+
# nuke the mirror directory since it is probably bogus (make this
|
455
|
+
# code fire off the next time checkout is done)
|
456
|
+
shell.say "Failed. Removing the [potentially corrupted] mirror.", :red
|
457
|
+
command = "rm -rf #{mirror_dir}"
|
458
|
+
shell.say command
|
459
|
+
bash_execute(command)
|
460
|
+
|
461
|
+
raise(GitHubError, 'Error while executing the following command: '\
|
462
|
+
"#{command}. Perhaps you need to configure "\
|
463
|
+
'global GitHub account settings for https '\
|
464
|
+
'authentication to work properly?')
|
465
|
+
end
|
466
|
+
else
|
467
|
+
raise(GitHubError, "Invalid GitHub protocol: \"#{github_protocol}\"")
|
468
|
+
end
|
589
469
|
end
|
590
|
-
shell.say "log entry: #{@svn_log}"
|
591
|
-
end
|
592
470
|
|
593
|
-
|
594
|
-
|
595
|
-
#
|
596
|
-
|
597
|
-
|
471
|
+
update_mirror
|
472
|
+
|
473
|
+
# ensure "work" directory is removed from worktree
|
474
|
+
remove
|
475
|
+
|
476
|
+
# create "work" directory with proper commit
|
477
|
+
shell.say "\nSetting up worktree repo...", :blue
|
478
|
+
FileUtils.mkdir_p mesa_dir
|
479
|
+
command = "git -C #{mirror_dir} worktree add #{mesa_dir} #{new_sha}"
|
480
|
+
shell.say command
|
481
|
+
return if bash_execute(command)
|
482
|
+
|
483
|
+
raise(GitHubError, 'Failed while executing the following command: '\
|
484
|
+
"\"#{command}\".")
|
598
485
|
end
|
599
486
|
|
600
|
-
def
|
601
|
-
|
487
|
+
def update_mirror
|
488
|
+
shell.say "\nFetching MESA history...", :blue
|
489
|
+
command = "git -C #{mirror_dir} fetch origin"
|
490
|
+
shell.say command
|
491
|
+
# fail loudly
|
492
|
+
return if bash_execute(command)
|
493
|
+
|
494
|
+
raise(GitHubError, 'Failed while executing the following command: '\
|
495
|
+
"\"#{command}\".")
|
602
496
|
end
|
603
497
|
|
604
|
-
def
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
498
|
+
def remove
|
499
|
+
return unless File.exist? mesa_dir
|
500
|
+
shell.say "\nRemoving work directory from worktree (clearing old data)...",
|
501
|
+
:blue
|
502
|
+
command = "git -C #{mirror_dir} worktree remove --force #{mesa_dir}"
|
503
|
+
shell.say command
|
504
|
+
return if bash_execute(command)
|
505
|
+
|
506
|
+
shell.say "Failed. Simply trying to remove the directory.", :red
|
507
|
+
command = "rm -rf #{mesa_dir}"
|
508
|
+
shell.say command
|
509
|
+
# fail loudly (the "true" tells bash_execute to raise an exception if
|
510
|
+
# the command fails)
|
511
|
+
bash_execute(command, true)
|
612
512
|
end
|
613
513
|
|
614
|
-
|
615
|
-
|
616
|
-
# match output of svn info to a line with the revision, capturing the
|
617
|
-
# number, and defaulting to 0 if none is found.
|
618
|
-
matches = /Revision\:\s+(\d+)/.match(`svn info #{mesa_dir}`)
|
619
|
-
unless matches.nil?
|
620
|
-
return matches[1].to_i
|
621
|
-
end
|
622
|
-
return 0
|
623
|
-
rescue Errno::ENOENT
|
624
|
-
return 0
|
514
|
+
def git_sha
|
515
|
+
bashticks("git -C #{mesa_dir} rev-parse HEAD")
|
625
516
|
end
|
626
517
|
|
627
|
-
|
628
|
-
|
629
|
-
contents = ''
|
630
|
-
File.open(File.join(mesa_dir, 'data', 'version_number'), 'r') do |f|
|
631
|
-
contents = f.read
|
632
|
-
end
|
633
|
-
contents.strip.to_i
|
518
|
+
def sha
|
519
|
+
git_sha
|
634
520
|
end
|
635
521
|
|
636
522
|
def clean
|
637
523
|
with_mesa_dir do
|
638
524
|
visit_and_check mesa_dir, MesaDirError, 'E\countered a problem in ' \
|
639
525
|
"running `clean` in #{mesa_dir}." do
|
640
|
-
|
641
|
-
|
526
|
+
shell.say('MESA_DIR = ' + ENV['MESA_DIR'])
|
527
|
+
shell.say './clean'
|
642
528
|
bash_execute('./clean')
|
643
529
|
end
|
644
530
|
end
|
@@ -649,8 +535,8 @@ class Mesa
|
|
649
535
|
with_mesa_dir do
|
650
536
|
visit_and_check mesa_dir, MesaDirError, 'Encountered a problem in ' \
|
651
537
|
"running `install` in #{mesa_dir}." do
|
652
|
-
|
653
|
-
|
538
|
+
shell.say('MESA_DIR = ' + ENV['MESA_DIR'])
|
539
|
+
shell.say './install'
|
654
540
|
bash_execute('./install')
|
655
541
|
end
|
656
542
|
end
|
@@ -661,20 +547,31 @@ class Mesa
|
|
661
547
|
|
662
548
|
# throw an error unless it seems like it's properly compiled
|
663
549
|
def check_installation
|
664
|
-
|
665
|
-
|
666
|
-
|
550
|
+
return if installed?
|
551
|
+
|
552
|
+
raise MesaDirError, 'Installation check failed (build.log doesn\'t '\
|
553
|
+
'show a successful installation).'
|
554
|
+
end
|
555
|
+
|
556
|
+
# sourced from $MESA_DIR/testhub.yml, which should be created after
|
557
|
+
# installation
|
558
|
+
def compiler_hash
|
559
|
+
data_file = File.join(mesa_dir, 'testhub.yml')
|
560
|
+
unless File.exist? data_file
|
561
|
+
raise(MesaDirError, "Could not find file testhub.yml in #{mesa_dir}.")
|
667
562
|
end
|
668
|
-
end
|
669
563
|
|
670
|
-
|
671
|
-
|
564
|
+
res = YAML.safe_load(File.read(data_file))
|
565
|
+
# currently version_number is reported, but we don't need that in Git land
|
566
|
+
res.delete('version_number') # returns the value, not the updated hash
|
567
|
+
res
|
672
568
|
end
|
673
569
|
|
674
570
|
## TEST SUITE METHODS
|
675
571
|
|
676
572
|
def check_mod(mod)
|
677
573
|
return if MesaTestCase.modules.include? mod
|
574
|
+
|
678
575
|
raise TestCaseDirError, "Invalid module: #{mod}. Must be one of: " +
|
679
576
|
MesaTestCase.modules.join(', ')
|
680
577
|
end
|
@@ -694,53 +591,32 @@ class Mesa
|
|
694
591
|
end
|
695
592
|
else
|
696
593
|
check_mod mod
|
697
|
-
# load data from the source file
|
698
|
-
source_lines = IO.readlines(
|
699
|
-
File.join(test_suite_dir(mod: mod), 'do1_test_source')
|
700
|
-
)
|
701
594
|
|
702
|
-
#
|
703
|
-
|
704
|
-
@
|
595
|
+
# convert output of +list_tests+ to a dictionary that maps
|
596
|
+
# names to numbers since +each_test_run+ only knows about numbers
|
597
|
+
@names_to_numbers[mod] = {}
|
598
|
+
@test_case_names[mod] = []
|
705
599
|
@test_cases[mod] = {}
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
710
|
-
|
711
|
-
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
photo: $4}
|
718
|
-
elsif line =~ one_skip
|
719
|
-
found_test = true
|
720
|
-
@test_data[mod][$1] = { success_string: $2, final_model: $3,
|
721
|
-
photo: nil }
|
722
|
-
elsif line =~ two_skip
|
723
|
-
found_test = true
|
724
|
-
@test_data[mod][$1] = { success_string: $2, final_model: nil,
|
725
|
-
photo: nil }
|
726
|
-
end
|
727
|
-
|
728
|
-
if found_test
|
729
|
-
@test_names[mod] << $1 unless @test_names[mod].include? $1
|
600
|
+
visit_dir(test_suite_dir(mod: mod), quiet: true) do
|
601
|
+
bashticks('./list_tests').split("\n").each do |line|
|
602
|
+
num, tc_name = line.strip.split
|
603
|
+
@names_to_numbers[mod][tc_name.strip] = num.to_i
|
604
|
+
@test_case_names[mod] << tc_name.strip
|
605
|
+
@test_cases[mod][tc_name.strip] = MesaTestCase.new(
|
606
|
+
test: tc_name.strip,
|
607
|
+
mod: mod,
|
608
|
+
position: num.to_i,
|
609
|
+
mesa: self
|
610
|
+
)
|
730
611
|
end
|
731
612
|
end
|
732
|
-
|
733
|
-
# make MesaTestCase objects accessible by name
|
734
|
-
@test_names[mod].each do |test_name|
|
735
|
-
data = @test_data[mod][test_name]
|
736
|
-
@test_cases[mod][test_name] = MesaTestCase.new(
|
737
|
-
test: test_name, mesa: self, success_string: data[:success_string],
|
738
|
-
mod: mod, final_model: data[:final_model], photo: data[:photo]
|
739
|
-
)
|
740
|
-
end
|
741
613
|
end
|
742
614
|
end
|
743
615
|
|
616
|
+
def test_case_count(mod: :all)
|
617
|
+
all_names_ordered(mod: mod).count
|
618
|
+
end
|
619
|
+
|
744
620
|
# can accept a number (in string form) as a name for indexed access
|
745
621
|
def find_test_case(test_case_name: nil, mod: :all)
|
746
622
|
if /\A[0-9]+\z/ =~ test_case_name
|
@@ -750,44 +626,16 @@ class Mesa
|
|
750
626
|
end
|
751
627
|
end
|
752
628
|
|
753
|
-
|
754
|
-
# revision 10000
|
755
|
-
def each_test_clean(mod: :all)
|
756
|
-
if mod == :all
|
757
|
-
MesaTestCase.modules.each { |this_mod| each_test_clean mod: this_mod }
|
758
|
-
else
|
759
|
-
check_mod mod
|
760
|
-
test_names[mod].each do |test_name|
|
761
|
-
test_cases[mod][test_name].clean
|
762
|
-
end
|
763
|
-
end
|
764
|
-
end
|
765
|
-
|
766
|
-
def each_test_run_and_diff(mod: :all, log_results: false)
|
629
|
+
def each_test_run(mod: :all)
|
767
630
|
check_installation
|
768
|
-
each_test_clean(mod: mod)
|
769
631
|
|
770
632
|
if mod == :all
|
771
633
|
MesaTestCase.modules.each do |this_mod|
|
772
|
-
|
634
|
+
each_test_run(mod: this_mod)
|
773
635
|
end
|
774
636
|
else
|
775
|
-
|
776
|
-
|
777
|
-
test_cases[mod][test_name].log_results if log_results
|
778
|
-
end
|
779
|
-
log_summary(mod: mod) if log_results
|
780
|
-
end
|
781
|
-
end
|
782
|
-
|
783
|
-
def each_test_load_results(mod: :all)
|
784
|
-
if mod == :all
|
785
|
-
MesaTestCase.modules.each do |this_mod|
|
786
|
-
each_test_load_results(mod: this_mod)
|
787
|
-
end
|
788
|
-
else
|
789
|
-
test_names[mod].each do |test_name|
|
790
|
-
test_cases[mod][test_name].load_results
|
637
|
+
visit_dir(test_suite_dir(mod: mod)) do
|
638
|
+
bash_execute('./each_test_run')
|
791
639
|
end
|
792
640
|
end
|
793
641
|
end
|
@@ -797,34 +645,21 @@ class Mesa
|
|
797
645
|
end
|
798
646
|
|
799
647
|
def installed?
|
800
|
-
#
|
801
|
-
#
|
802
|
-
|
803
|
-
|
804
|
-
|
805
|
-
# SOME_MODULE", after which the "SOME_MODULE" will be stored in $1
|
806
|
-
# that is the last module to be compiled by ./install.
|
807
|
-
IO.readlines(install_file).select do |line|
|
808
|
-
line =~ /^\s*do_one\w*\s+\w+/
|
809
|
-
end.last =~ /^\s*do_one\w*\s+(\w+)/
|
810
|
-
# module is "installed" if there is a nonzero number of files in the
|
811
|
-
# module's make directory of the form SOMETHING.mod
|
812
|
-
!Dir.entries(File.join(mesa_dir, $1, 'make')).select do |file|
|
813
|
-
File.extname(file) == '.mod'
|
814
|
-
end.empty?
|
648
|
+
# assume build log reflects installation status; does not account for
|
649
|
+
# mucking with modules after the fact
|
650
|
+
downloaded? && File.read(File.join(mesa_dir, 'build.log')).include?(
|
651
|
+
'MESA installation was successful'
|
652
|
+
)
|
815
653
|
end
|
816
654
|
|
817
|
-
|
818
655
|
private
|
819
656
|
|
820
|
-
# verify that mesa_dir is valid by checking for
|
821
|
-
# directory
|
657
|
+
# verify that mesa_dir is valid by checking for existence of test_suite
|
658
|
+
# directory for each module (somewhat arbitrary)
|
822
659
|
def check_mesa_dir
|
823
|
-
|
824
|
-
|
825
|
-
res &&= dir_or_symlink_exists?(test_suite_dir(mod: mod))
|
660
|
+
MesaTestCase.modules.inject(true) do |res, mod|
|
661
|
+
res && dir_or_symlink_exists?(test_suite_dir(mod: mod))
|
826
662
|
end
|
827
|
-
res
|
828
663
|
end
|
829
664
|
|
830
665
|
# change MESA_DIR for the execution of the block and then revert to the
|
@@ -845,49 +680,64 @@ class Mesa
|
|
845
680
|
end
|
846
681
|
end
|
847
682
|
|
848
|
-
def
|
683
|
+
def all_names_ordered(mod: :all)
|
684
|
+
load_test_source_data unless @names_to_numbers
|
849
685
|
if mod == :all
|
850
|
-
|
851
|
-
|
686
|
+
# build up list by first constructing each modules list and then
|
687
|
+
# concatenating them
|
688
|
+
MesaTestCase.modules.inject([]) do |res, this_mod|
|
689
|
+
res += all_names_ordered(mod: this_mod)
|
852
690
|
end
|
853
691
|
else
|
854
692
|
check_mod mod
|
855
|
-
res = []
|
856
|
-
|
857
|
-
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
'failure_type' => test_case.failure_type,
|
862
|
-
'success_type' => test_case.success_type,
|
863
|
-
'runtime_seconds' => test_case.runtime_seconds,
|
864
|
-
'omp_num_threads' => test_case.test_omp_num_threads,
|
865
|
-
'mesa_version' => test_case.mesa_version
|
866
|
-
}
|
867
|
-
end
|
868
|
-
summary_file = File.join(test_suite_dir(mod: mod), 'test_summary.yml')
|
869
|
-
File.open(summary_file, 'w') do |f|
|
870
|
-
f.write(YAML.dump(res))
|
693
|
+
res = Array.new(@names_to_numbers[mod].length, '')
|
694
|
+
|
695
|
+
# values of the hash give their order, keys are the names, so
|
696
|
+
# we assign keys to positions in the array according to their value
|
697
|
+
@names_to_numbers[mod].each_pair do |key, val|
|
698
|
+
res[val - 1] = key # +list_tests+ gives 1-indexed positions
|
871
699
|
end
|
700
|
+
res
|
872
701
|
end
|
873
702
|
end
|
874
703
|
|
875
704
|
def find_test_case_by_name(test_case_name: nil, mod: :all)
|
705
|
+
load_test_source_data unless @names_to_numbers
|
876
706
|
if mod == :all
|
877
|
-
# look through all loaded modules for desired test case name,
|
878
|
-
#
|
879
|
-
|
880
|
-
|
881
|
-
|
707
|
+
# look through all loaded modules for desired test case name, only
|
708
|
+
# return a test case if a single case is found with that name
|
709
|
+
case all_names_ordered.count(test_case_name)
|
710
|
+
when 1
|
711
|
+
# it exists in exactly one module, but we need to find the module
|
712
|
+
# and then return the +MesaTestCase+ object
|
713
|
+
MesaTestCase.modules.each do |this_mod|
|
714
|
+
if @test_case_names[this_mod].include? test_case_name
|
715
|
+
# found it, return the appropriate object
|
716
|
+
return @test_cases[this_mod][test_case_name]
|
717
|
+
end
|
882
718
|
end
|
719
|
+
raise 'Weird problem: found test case in overall names, but '\
|
720
|
+
"not in any particular module. This shouldn't happen."
|
721
|
+
when 0
|
722
|
+
raise(TestCaseDirError, "Could not find test case #{test_case_name} "\
|
723
|
+
'in any module.')
|
724
|
+
else
|
725
|
+
raise(TestCaseDirError, 'Found multiple test cases named '\
|
726
|
+
"#{test_case_name} in multiple modules. Indicate the module you "\
|
727
|
+
'want to search.')
|
883
728
|
end
|
884
|
-
#
|
885
|
-
nil
|
729
|
+
# append this array to the end of the exisitng one
|
886
730
|
else
|
887
731
|
# module specified; check it and return the proper test case (may be nil
|
888
732
|
# if the test case doesn't exist)
|
889
733
|
check_mod mod
|
890
|
-
@
|
734
|
+
if @test_case_names[mod].include? test_case_name
|
735
|
+
# happy path: test case exists in the specified module
|
736
|
+
return @test_cases[mod][test_case_name]
|
737
|
+
else
|
738
|
+
raise TestCaseDirError.new('Could not find test case ' \
|
739
|
+
"#{test_case_name} in the #{mod} module.")
|
740
|
+
end
|
891
741
|
end
|
892
742
|
end
|
893
743
|
|
@@ -895,149 +745,69 @@ class Mesa
|
|
895
745
|
# this will be the index in the name array of the proper module of
|
896
746
|
# the desired test case
|
897
747
|
# input numbers are 1-indexed, but we'll fix that later
|
898
|
-
|
899
|
-
|
748
|
+
if test_number < 1 || test_number > test_case_count(mod: mod)
|
749
|
+
raise TestCaseDirError.new('Invalid test case number for searching '\
|
750
|
+
"in module #{mod}. Must be between 1 and #{test_case_count(mod: mod)}.")
|
751
|
+
end
|
900
752
|
|
901
753
|
if mod == :all
|
902
|
-
#
|
903
|
-
|
904
|
-
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
test_case_name: @test_names[this_mod][i - 1],
|
913
|
-
mod: this_mod
|
754
|
+
# can get the name easily, now need to find the module
|
755
|
+
test_case_name = all_names_ordered[test_number - 1]
|
756
|
+
MesaTestCase.modules.each do |mod|
|
757
|
+
if test_number <= test_case_count(mod: mod)
|
758
|
+
# test must live in this module; we have everything
|
759
|
+
return MesaTestCase.new(
|
760
|
+
test: test_case_name,
|
761
|
+
mod: mod,
|
762
|
+
mesa: self,
|
763
|
+
position: @names_to_numbers[mod][test_case_name]
|
914
764
|
)
|
765
|
+
else
|
766
|
+
# number was too big, so decrement by this modules case count
|
767
|
+
# and move on to next one
|
768
|
+
test_number -= test_case_count(mod: mod)
|
915
769
|
end
|
916
|
-
# index lies outside possible range for this module, move on to
|
917
|
-
# next module and decrement index by the number of test cases in this
|
918
|
-
# module
|
919
|
-
i -= @test_names[this_mod].length
|
920
770
|
end
|
921
|
-
# return
|
922
|
-
|
771
|
+
# should return before we get here, but fail hard if we do
|
772
|
+
raise TestCaseDirError.new('Unknown problem in loading test case #' +
|
773
|
+
test_number + '.')
|
923
774
|
else
|
924
|
-
# module was specified, so
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
775
|
+
# module was specified, so we can get at everything right away
|
776
|
+
check_mod mod
|
777
|
+
return MesaTestCase.new(
|
778
|
+
test: all_names_ordered(mod: mod)[test_number - 1],
|
779
|
+
mod: mod,
|
780
|
+
mesa: self,
|
781
|
+
position: test_number
|
931
782
|
)
|
932
783
|
end
|
933
784
|
end
|
934
785
|
end
|
935
786
|
|
936
787
|
class MesaTestCase
|
937
|
-
attr_reader :test_name, :
|
938
|
-
:failure_msg, :success_msg, :photo, :runtime_seconds,
|
939
|
-
:test_omp_num_threads, :mesa_version, :shell, :mod, :retries,
|
940
|
-
:backups, :steps, :runtime_minutes, :summary_text, :compiler,
|
941
|
-
:compiler_version, :diff, :checksum, :rn_mem, :re_mem,
|
942
|
-
:re_time, :total_runtime_seconds
|
943
|
-
attr_accessor :data_names, :data_types, :failure_type, :success_type,
|
944
|
-
:outcome
|
788
|
+
attr_reader :test_name, :mesa, :mod, :position, :shell
|
945
789
|
|
946
790
|
def self.modules
|
947
|
-
%i[star binary]
|
791
|
+
%i[star binary astero]
|
948
792
|
end
|
949
793
|
|
950
|
-
def initialize(test: nil, mesa: nil,
|
951
|
-
final_model: 'final.mod', photo: nil, mod: nil)
|
794
|
+
def initialize(test: nil, mesa: nil, mod: nil, position: nil)
|
952
795
|
@test_name = test
|
953
|
-
@mesa_dir = mesa.mesa_dir
|
954
796
|
@mesa = mesa
|
955
|
-
@mesa_version = mesa.version_number
|
956
|
-
@success_string = success_string
|
957
|
-
@final_model = final_model
|
958
|
-
@photo = photo
|
959
|
-
@failure_type = nil
|
960
|
-
@success_type = nil
|
961
|
-
@outcome = :not_tested
|
962
|
-
@runtime_seconds = 0
|
963
|
-
@test_omp_num_threads = 1
|
964
|
-
@runtime_minutes = 0
|
965
|
-
@total_runtime_seconds = 0
|
966
|
-
@retries = 0
|
967
|
-
@backups = 0
|
968
|
-
@steps = 0
|
969
|
-
# 2 (default) means uknown. Updated by running or loading data.
|
970
|
-
# 1 means did diffs (not update_checksums; like each_test_run_and_diff)
|
971
|
-
# 0 means no diffs (update_checksums; like each_test_run)
|
972
|
-
@diff = 2
|
973
|
-
# start with nil. Should only be updated to a non-nil value if test is
|
974
|
-
# completely successful
|
975
|
-
@checksum = nil
|
976
|
-
@re_time = nil # rn_time is in the form of @runtime_seconds
|
977
|
-
|
978
|
-
# these only get used with modern versions of both the sdk and the test
|
979
|
-
# suite
|
980
|
-
@rn_mem = nil
|
981
|
-
@re_mem = nil
|
982
|
-
|
983
|
-
# note: this gets overridden for new runs, so this is probably irrelevant
|
984
|
-
@summary_text = nil
|
985
|
-
|
986
|
-
# this overrides the submitters choice if it is non-nil
|
987
|
-
@compiler = mesa.using_sdk ? 'SDK' : nil
|
988
|
-
# only relevant if @compiler is SDK. Gets set during do_one
|
989
|
-
@compiler_version = nil
|
990
|
-
|
991
797
|
unless MesaTestCase.modules.include? mod
|
992
798
|
raise TestCaseDirError, "Invalid module: #{mod}. Must be one of: " +
|
993
799
|
MesaTestCase.modules.join(', ')
|
994
800
|
end
|
995
801
|
@mod = mod
|
996
|
-
@
|
997
|
-
run_test_string: "#{test_name} run failed: does not match test string",
|
998
|
-
final_model: "#{test_name} run failed: final model #{final_model} not " \
|
999
|
-
'made.',
|
1000
|
-
run_checksum: "#{test_name} run failed: checksum for #{final_model} " \
|
1001
|
-
'does not match after ./rn',
|
1002
|
-
run_diff: "#{test_name} run failed: diff #{final_model} " \
|
1003
|
-
'final_check.mod after ./rn',
|
1004
|
-
photo_file: "#{test_name} restart failed: #{photo} does not exist",
|
1005
|
-
photo_checksum: "#{test_name} restart failed: checksum for " \
|
1006
|
-
"#{final_model} does not match after ./re",
|
1007
|
-
photo_diff: "#{test_name} restart failed: diff #{final_model} " \
|
1008
|
-
'final_check.mod after ./re',
|
1009
|
-
compilation: "#{test_name} compilation failed"
|
802
|
+
@position = position
|
1010
803
|
|
1011
|
-
|
1012
|
-
@
|
1013
|
-
run_test_string: "#{test_name} run: found test string: " \
|
1014
|
-
"'#{success_string}'",
|
1015
|
-
run_checksum: "#{test_name} run: checksum for #{final_model} matches " \
|
1016
|
-
'after ./rn',
|
1017
|
-
photo_checksum: "#{test_name} restart: checksum for #{final_model} " \
|
1018
|
-
"matches after ./re #{photo}"
|
1019
|
-
}
|
804
|
+
# way to output colored text to shell
|
805
|
+
@shell = Thor::Shell::Color.new
|
1020
806
|
|
1021
807
|
# validate stuff
|
1022
808
|
check_mesa_dir
|
1023
809
|
check_test_case
|
1024
810
|
|
1025
|
-
@data = {}
|
1026
|
-
@data_names = []
|
1027
|
-
|
1028
|
-
# way to output colored text to shell
|
1029
|
-
@shell = Thor::Shell::Color.new
|
1030
|
-
end
|
1031
|
-
|
1032
|
-
def passed?
|
1033
|
-
if @outcome == :pass
|
1034
|
-
true
|
1035
|
-
elsif @outcome == :fail
|
1036
|
-
false
|
1037
|
-
else
|
1038
|
-
raise TestCaseDirError, 'Cannot determine pass/fail status of ' \
|
1039
|
-
"#{test_name} yet."
|
1040
|
-
end
|
1041
811
|
end
|
1042
812
|
|
1043
813
|
def test_suite_dir
|
@@ -1048,214 +818,26 @@ class MesaTestCase
|
|
1048
818
|
File.join(test_suite_dir, test_name)
|
1049
819
|
end
|
1050
820
|
|
1051
|
-
|
1052
|
-
|
1053
|
-
raise InvalidDataType, "Invalid data type: #{datum_type}. Must be one "\
|
1054
|
-
'of ' + data_types.join(', ') + '.'
|
1055
|
-
end
|
1056
|
-
@data[datum_name] = datum_type
|
1057
|
-
@data_names << datum_name
|
1058
|
-
end
|
1059
|
-
|
1060
|
-
def omp_num_threads
|
1061
|
-
ENV['OMP_NUM_THREADS'].to_i || 1
|
1062
|
-
end
|
1063
|
-
|
1064
|
-
# based on $MESA_DIR/star/test_suite/each_test_clean, revision 10000
|
1065
|
-
def clean
|
1066
|
-
shell.say("Cleaning #{test_name}", color = :yellow)
|
1067
|
-
# puts ''
|
1068
|
-
check_mesa_dir
|
1069
|
-
check_test_case
|
1070
|
-
in_dir do
|
1071
|
-
puts './clean'
|
1072
|
-
unless bash_execute('./clean')
|
1073
|
-
raise TestCaseDirError, 'Encountered an error while running ./clean ' \
|
1074
|
-
"in #{Dir.getwd}."
|
1075
|
-
end
|
1076
|
-
shell.say 'Removing all files from LOGS, LOGS1, LOGS2, photos, ' \
|
1077
|
-
'photos1, and photos2 as well as old test results', color = :blue
|
1078
|
-
FileUtils.rm_f Dir.glob('LOGS/*')
|
1079
|
-
FileUtils.rm_f Dir.glob('LOGS1/*')
|
1080
|
-
FileUtils.rm_f Dir.glob('LOGS2/*')
|
1081
|
-
FileUtils.rm_f Dir.glob('photos/*')
|
1082
|
-
FileUtils.rm_f Dir.glob('photos1/*')
|
1083
|
-
FileUtils.rm_f Dir.glob('photos2/*')
|
1084
|
-
|
1085
|
-
shell.say 'Removing files binary_history.data, out.txt, ' \
|
1086
|
-
'test_results.yml, and memory usage files', color = :blue
|
1087
|
-
FileUtils.rm_f 'binary_history.data'
|
1088
|
-
FileUtils.rm_f 'out.txt'
|
1089
|
-
FileUtils.rm_f 'test_results.yml'
|
1090
|
-
FileUtils.rm_f Dir.glob('mem-*.txt')
|
1091
|
-
if File.directory? File.join('star_history', 'history_out')
|
1092
|
-
shell.say 'Removing all files of the form history_out* from ' \
|
1093
|
-
'star_history', :blue
|
1094
|
-
FileUtils.rm_f Dir.glob(File.join('star_history', 'history_out', '*'))
|
1095
|
-
end
|
1096
|
-
if File.directory? File.join('star_profile', 'profiles_out')
|
1097
|
-
shell.say 'Removing all files of the form profiles_out* from ' \
|
1098
|
-
'star_profile', color = :blue
|
1099
|
-
FileUtils.rm_f Dir.glob(File.join('star_profile', 'profiles_out', '*'))
|
1100
|
-
end
|
1101
|
-
shell.say 'Removing .running', color = :blue
|
1102
|
-
FileUtils.rm_f '.running'
|
1103
|
-
end
|
1104
|
-
end
|
1105
|
-
|
1106
|
-
# based on $MESA_DIR/star/test_suite/each_test_run_and_diff, revision 10000
|
821
|
+
# just punt to +each_test_run+ in the test_suite directory. It's your problem
|
822
|
+
# now, sucker!
|
1107
823
|
def do_one
|
1108
824
|
shell.say("Testing #{test_name}", :yellow)
|
1109
|
-
|
1110
|
-
|
1111
|
-
@test_omp_num_threads = omp_num_threads
|
1112
|
-
if mesa.using_sdk
|
1113
|
-
version_bin = File.join(ENV['MESASDK_ROOT'], 'bin', 'mesasdk_version')
|
1114
|
-
# can't use bash_execute because the return value of bash_execute is the
|
1115
|
-
# exit status of the commmand (true or false), whereas backticks give the
|
1116
|
-
# output (the version string) as the output
|
1117
|
-
if File.exist? version_bin
|
1118
|
-
# newer SDKs have a simple executable
|
1119
|
-
@compiler_version = `#{version_bin}`.strip
|
1120
|
-
else
|
1121
|
-
# older way, call bash on it (file is mesasdk_version.sh)
|
1122
|
-
@compiler_version = `bash -c #{version_bin + '.sh'}`.strip
|
1123
|
-
end
|
1124
|
-
|
1125
|
-
shell.say("Using version #{@compiler_version} of the SDK.", :blue)
|
1126
|
-
end
|
1127
|
-
in_dir do
|
1128
|
-
FileUtils.touch '.running'
|
1129
|
-
build_and_run
|
1130
|
-
# report memory usage if it is available
|
1131
|
-
if File.exist?('mem-rn.txt')
|
1132
|
-
@rn_mem = File.read('mem-rn.txt').strip.to_i
|
1133
|
-
end
|
1134
|
-
if File.exist?('mem-re.txt')
|
1135
|
-
@re_mem = File.read('mem-re.txt').strip.to_i
|
1136
|
-
end
|
1137
|
-
FileUtils.rm '.running'
|
1138
|
-
puts ''
|
825
|
+
visit_dir(test_suite_dir) do
|
826
|
+
bash_execute("./each_test_run #{position}")
|
1139
827
|
end
|
1140
|
-
test_finish = Time.now
|
1141
|
-
@total_runtime_seconds = (test_finish - test_start).to_i
|
1142
828
|
end
|
1143
829
|
|
1144
|
-
def
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
res = {
|
1150
|
-
'test_case' => test_name,
|
1151
|
-
'module' => mod,
|
1152
|
-
'runtime_seconds' => runtime_seconds,
|
1153
|
-
're_time' => re_time,
|
1154
|
-
'total_runtime_seconds' => total_runtime_seconds,
|
1155
|
-
'mesa_version' => mesa_version,
|
1156
|
-
'outcome' => outcome,
|
1157
|
-
'omp_num_threads' => test_omp_num_threads,
|
1158
|
-
'success_type' => success_type,
|
1159
|
-
'failure_type' => failure_type,
|
1160
|
-
'runtime_minutes' => runtime_minutes,
|
1161
|
-
'retries' => retries,
|
1162
|
-
'backups' => backups,
|
1163
|
-
'steps' => steps,
|
1164
|
-
'diff' => diff,
|
1165
|
-
'checksum' => checksum,
|
1166
|
-
'rn_mem' => rn_mem,
|
1167
|
-
're_mem' => re_mem,
|
1168
|
-
'summary_text' => summary_text
|
1169
|
-
}
|
1170
|
-
if compiler == 'SDK'
|
1171
|
-
res['compiler'] = 'SDK'
|
1172
|
-
res['compiler_version'] = compiler_version
|
830
|
+
def results_hash
|
831
|
+
testhub_file = File.join(test_case_dir, 'testhub.yml')
|
832
|
+
unless File.exist?(testhub_file)
|
833
|
+
raise TestCaseDirError.new('No results found for test case '\
|
834
|
+
"#{test_name}.")
|
1173
835
|
end
|
1174
|
-
File.
|
1175
|
-
shell.say "Successfully saved results to file #{save_file}.\n", :green
|
1176
|
-
end
|
1177
|
-
|
1178
|
-
def load_results
|
1179
|
-
# loads all parameters from a previous test run, likely for submission
|
1180
|
-
# purposes
|
1181
|
-
load_file = File.join(test_case_dir, 'test_results.yml')
|
1182
|
-
shell.say "Loading data from #{load_file}...", :blue
|
1183
|
-
unless File.exist? load_file
|
1184
|
-
shell.say "No such file: #{load_file}. No data loaded.", :red
|
1185
|
-
return
|
1186
|
-
end
|
1187
|
-
data = YAML.safe_load(File.read(load_file), [Symbol])
|
1188
|
-
@runtime_seconds = data['runtime_seconds'] || @runtime_seconds
|
1189
|
-
@re_time = data['re_time'] || @re_time
|
1190
|
-
@total_runtime_seconds = data['total_runtime_seconds'] || @total_runtime_seconds
|
1191
|
-
@mod = data['module'] || @mod
|
1192
|
-
@mesa_version = data['mesa_version'] || @mesa_version
|
1193
|
-
@outcome = data['outcome'] || @outcome
|
1194
|
-
@test_omp_num_threads = data['omp_num_threads'] || @test_omp_num_threads
|
1195
|
-
@success_type = data['success_type'] || @success_type
|
1196
|
-
@failure_type = data['failure_type'] || @failure_type
|
1197
|
-
@runtime_minutes = data['runtime_minutes'] || @runtime_minutes
|
1198
|
-
@retries = data['retries'] || @retries
|
1199
|
-
@backups = data['backups'] || @backups
|
1200
|
-
@steps = data['steps'] || @steps
|
1201
|
-
@diff = data['diff'] || @diff
|
1202
|
-
@checksum = data['checksum'] || @checksum
|
1203
|
-
@rn_mem = data['rn_mem'] || @rn_mem
|
1204
|
-
@re_mem = data['re_mem'] || @re_mem
|
1205
|
-
@summary_text = data['summary_text'] || @summary_text
|
1206
|
-
@compiler = data['compiler'] || @compiler
|
1207
|
-
|
1208
|
-
@compiler_version = data['compiler_version'] || @compiler_version
|
1209
|
-
|
1210
|
-
# convert select data to symbols since that is how they are used
|
1211
|
-
@outcome = @outcome.to_sym if @outcome
|
1212
|
-
@success_type = @success_type.to_sym if @success_type
|
1213
|
-
@failure_type = @failure_type.to_sym if @failure_type
|
1214
|
-
|
1215
|
-
shell.say "Done loading data from #{load_file}.\n", :green
|
1216
|
-
end
|
1217
|
-
|
1218
|
-
def load_summary_data
|
1219
|
-
begin
|
1220
|
-
out_data = parse_out
|
1221
|
-
@summary_text = get_summary_text
|
1222
|
-
rescue Errno::ENOENT
|
1223
|
-
shell.say "\nError loading data from #{out_file}. No summary data "\
|
1224
|
-
'loaded. Proceeding anyway.', :red
|
1225
|
-
else
|
1226
|
-
@runtime_minutes = out_data[:runtime_minutes]
|
1227
|
-
@retries = out_data[:retries]
|
1228
|
-
@backups = out_data[:backups]
|
1229
|
-
@steps = out_data[:steps]
|
1230
|
-
end
|
1231
|
-
end
|
1232
|
-
|
1233
|
-
def parse_out
|
1234
|
-
runtime_minutes = 0
|
1235
|
-
retries = 0
|
1236
|
-
backups = 0
|
1237
|
-
steps = 0
|
1238
|
-
run_summaries.each do |summary|
|
1239
|
-
summary =~ /^\s*runtime\s*\(minutes\),\s+retries,\s+backups,\ssteps\s+(\d+\.?\d*)\s+(\d+)\s+(\d+)\s+(\d+)\s*$/
|
1240
|
-
runtime_minutes += $1.to_f
|
1241
|
-
retries += $2.to_i
|
1242
|
-
backups += $3.to_i
|
1243
|
-
steps += $4.to_i
|
1244
|
-
end
|
1245
|
-
{runtime_minutes: runtime_minutes, retries: retries, backups: backups,
|
1246
|
-
steps: steps}
|
836
|
+
YAML.safe_load(File.read(testhub_file), [Symbol])
|
1247
837
|
end
|
1248
838
|
|
1249
839
|
private
|
1250
840
|
|
1251
|
-
def data_types
|
1252
|
-
%i[float integer string boolean]
|
1253
|
-
end
|
1254
|
-
|
1255
|
-
def rn_time
|
1256
|
-
runtime_seconds
|
1257
|
-
end
|
1258
|
-
|
1259
841
|
# cd into the test case directory, do something in a block, then cd back
|
1260
842
|
# to original directory
|
1261
843
|
def in_dir(&block)
|
@@ -1269,322 +851,10 @@ class MesaTestCase
|
|
1269
851
|
raise TestCaseDirError, "No such test case: #{test_case_dir}."
|
1270
852
|
end
|
1271
853
|
|
1272
|
-
# verify that mesa_dir is valid by checking for
|
1273
|
-
# directory
|
854
|
+
# "verify" that mesa_dir is valid by checking for test_suite directory
|
1274
855
|
def check_mesa_dir
|
1275
|
-
is_valid =
|
1276
|
-
|
1277
|
-
raise MesaDirError, "Invalid MESA dir: #{mesa_dir}" unless is_valid
|
1278
|
-
end
|
1279
|
-
|
1280
|
-
# append message to log file
|
1281
|
-
def log_message(msg, color = nil, log_file = 'out.txt')
|
1282
|
-
if color.nil?
|
1283
|
-
shell.say(msg)
|
1284
|
-
else
|
1285
|
-
shell.say(msg, color)
|
1286
|
-
end
|
1287
|
-
File.open(log_file, 'a') { |f| f.puts(msg) }
|
1288
|
-
end
|
1289
|
-
|
1290
|
-
# write failure message to log file
|
1291
|
-
def write_failure_message
|
1292
|
-
msg = "******************** #{failure_msg[@failure_type]} " \
|
1293
|
-
'********************'
|
1294
|
-
log_message(msg, :red)
|
1295
|
-
end
|
1296
|
-
|
1297
|
-
# write success message to log file
|
1298
|
-
def write_success_msg(success_type)
|
1299
|
-
msg = 'PASS ' + success_msg[success_type]
|
1300
|
-
log_message(msg, :green)
|
1301
|
-
end
|
1302
|
-
|
1303
|
-
# used as return value for run or photo test. Logs failure to text file, and
|
1304
|
-
# sets internal status to failing
|
1305
|
-
def fail_test(failure_type)
|
1306
|
-
@failure_type = failure_type
|
1307
|
-
@outcome = :fail
|
1308
|
-
write_failure_message
|
1309
|
-
false
|
1310
|
-
end
|
1311
|
-
|
1312
|
-
# used as return value for run or photo test. Logs data to text file, and
|
1313
|
-
# sets internal status to passing
|
1314
|
-
def succeed(success_type)
|
1315
|
-
@success_type = success_type
|
1316
|
-
@outcome = :pass
|
1317
|
-
# this should ONLY be read after we are certain we've passed AND that we
|
1318
|
-
# even have a newly-made checksum
|
1319
|
-
if File.exist?('checks.md5') && @mesa.update_checksums
|
1320
|
-
# sometimes we want to ignore the final checksum value
|
1321
|
-
# we still want to check that restarts on individual machines are bit-for-bit
|
1322
|
-
# but we don't require bit-for-bit globally, so we report a bogus checksum
|
1323
|
-
if File.exist?('.ignore_checksum') then
|
1324
|
-
@checksum ='00000000000000000000000000000000'
|
1325
|
-
else
|
1326
|
-
@checksum = File.read('checks.md5').split.first
|
1327
|
-
end
|
1328
|
-
end
|
1329
|
-
write_success_msg(success_type)
|
1330
|
-
true
|
1331
|
-
end
|
1332
|
-
|
1333
|
-
def check_run
|
1334
|
-
# assumes we are in the directory already, called from something else
|
1335
|
-
run_start = Time.now
|
1336
|
-
|
1337
|
-
# do the run
|
1338
|
-
rn_command = if ENV['MESASDK_ROOT'] && File.exist?(File.join(ENV['MESASDK_ROOT'], 'bin', 'time'))
|
1339
|
-
%q(command time -f '%M' -o mem-rn.txt ./rn > out.txt 2> ) +
|
1340
|
-
'err.txt'
|
1341
|
-
else
|
1342
|
-
'./rn >> out.txt 2> err.txt'
|
1343
|
-
end
|
1344
|
-
puts rn_command
|
1345
|
-
bash_execute(rn_command)
|
1346
|
-
|
1347
|
-
# report runtime and clean up
|
1348
|
-
run_finish = Time.now
|
1349
|
-
@runtime_seconds = (run_finish - run_start).to_i
|
1350
|
-
@rn_time = (run_finish - run_start).to_i
|
1351
|
-
shell.say("Finished with ./rn; runtime = #{@runtime_seconds} seconds.",
|
1352
|
-
:blue)
|
1353
|
-
append_and_rm_err
|
1354
|
-
|
1355
|
-
# look for success text
|
1356
|
-
success = true
|
1357
|
-
File.open('out.txt', 'r') do |f|
|
1358
|
-
success = !f.read.downcase.scan(success_string.downcase).empty?
|
1359
|
-
end
|
1360
|
-
# bail if there was no test string found
|
1361
|
-
return fail_test(:run_test_string) unless success
|
1362
|
-
|
1363
|
-
# no final model to check, and we already found the test string, so pass
|
1364
|
-
return succeed(:run_test_string) unless final_model
|
1365
|
-
|
1366
|
-
# display runtime message
|
1367
|
-
puts IO.readlines('out.txt').select { |line| line.scan(/runtime/i) }[-1]
|
1368
|
-
|
1369
|
-
# there's supposed to be a final model; check that it exists first
|
1370
|
-
return fail_test(:final_model) unless File.exist?(final_model)
|
1371
|
-
|
1372
|
-
# update checksums
|
1373
|
-
#
|
1374
|
-
# if this is true, behave like each_test_run. update the checksum
|
1375
|
-
# after rn and then check it matches after re
|
1376
|
-
#
|
1377
|
-
# if this is false, behave like each_test_run_and_diff. assume
|
1378
|
-
# the checksum is up-to-date and check it matches after rn and re.
|
1379
|
-
if @mesa.update_checksums
|
1380
|
-
@diff = 0 # this means no diffs run
|
1381
|
-
puts "md5sum \"#{final_model}\" > checks.md5"
|
1382
|
-
bash_execute("md5sum \"#{final_model}\" > checks.md5")
|
1383
|
-
FileUtils.cp final_model, 'final_check.mod'
|
1384
|
-
|
1385
|
-
# if there's no photo, we won't check the checksum, so we've succeeded
|
1386
|
-
return succeed(:run_test_string) unless photo
|
1387
|
-
# if there is a photo, we'll have to wait and see
|
1388
|
-
return true
|
1389
|
-
end
|
1390
|
-
|
1391
|
-
# check that final model matches
|
1392
|
-
@diff = 1 # this means diffs were run
|
1393
|
-
puts './ck >& final_check_diff.txt'
|
1394
|
-
return fail_test(:run_checksum) unless
|
1395
|
-
bash_execute('./ck >& final_check_diff.txt')
|
1396
|
-
return fail_test(:run_diff) if File.exist?('final_check_diff.txt') &&
|
1397
|
-
!File.read('final_check_diff.txt').empty?
|
1398
|
-
return succeed(:run_checksum) if File.exist? final_model
|
1399
|
-
end
|
1400
|
-
|
1401
|
-
# prepare for and do restart, check results, and return pass/fail status
|
1402
|
-
def check_restart
|
1403
|
-
# abort if there is not photo specified
|
1404
|
-
return unless photo
|
1405
|
-
|
1406
|
-
# get penultimate photo
|
1407
|
-
if photo == "auto" then
|
1408
|
-
# get all photos [single-star (x100) or binary (b_x100); exclude binary stars (1_x100, 2_x100)]
|
1409
|
-
photo_files = Dir["photos/*"].select{|p| p =~ /^photos\/(b_)?x?\d+$/}
|
1410
|
-
# pull out 2nd most recent one
|
1411
|
-
re_photo = File.basename(photo_files.sort_by { |file_name| File.stat(file_name).mtime } [-2])
|
1412
|
-
# if binary, trim off prefix
|
1413
|
-
re_photo = re_photo.sub("b_", "")
|
1414
|
-
else
|
1415
|
-
re_photo = photo
|
1416
|
-
end
|
1417
|
-
|
1418
|
-
# check that photo file actually exists
|
1419
|
-
unless File.exist?(File.join('photos', re_photo)) ||
|
1420
|
-
File.exist?(File.join('photos1', re_photo)) ||
|
1421
|
-
File.exist?(File.join('photos', "b_#{re_photo}"))
|
1422
|
-
return fail_test(:photo_file)
|
1423
|
-
end
|
1424
|
-
|
1425
|
-
# remove final model since it will be remade by restart
|
1426
|
-
FileUtils.rm_f final_model
|
1427
|
-
|
1428
|
-
# do restart and consolidate output. Command depends on if we have access
|
1429
|
-
# to SDK version of gnu time.
|
1430
|
-
re_command = if ENV['MESASDK_ROOT'] && File.exist?(File.join(ENV['MESASDK_ROOT'], 'bin', 'time'))
|
1431
|
-
%q(command time -f '%M' -o mem-re.txt ./re ) + "#{re_photo}" \
|
1432
|
-
' >> out.txt 2> err.txt'
|
1433
|
-
else
|
1434
|
-
"./re #{re_photo} >> out.txt 2> err.txt"
|
1435
|
-
end
|
1436
|
-
|
1437
|
-
puts re_command
|
1438
|
-
# puts "./re #{re_photo} >> out.txt 2> err.txt"
|
1439
|
-
re_start = Time.now
|
1440
|
-
# bash_execute("./re #{re_photo} >> out.txt 2> err.txt")
|
1441
|
-
# bash_execute(%Q{command time -f '%M' -o mem-re.txt ./re #{re_photo} >> out.txt 2> err.txt})
|
1442
|
-
bash_execute(re_command)
|
1443
|
-
re_finish = Time.now
|
1444
|
-
@re_time = (re_finish - re_start).to_i
|
1445
|
-
append_and_rm_err
|
1446
|
-
|
1447
|
-
# check that final model matches
|
1448
|
-
puts './ck >& final_check_diff.txt'
|
1449
|
-
return fail_test(:photo_checksum) unless
|
1450
|
-
bash_execute('./ck >& final_check_diff.txt')
|
1451
|
-
return fail_test(:photo_diff) if
|
1452
|
-
File.exist?('final_check_diff.txt') &&
|
1453
|
-
!File.read('final_check_diff.txt').empty?
|
1454
|
-
succeed(:photo_checksum)
|
1455
|
-
end
|
1456
|
-
|
1457
|
-
def build_and_run
|
1458
|
-
# assumes we are in the test case directory. Should only be called
|
1459
|
-
# in the context of an `in_dir` block.
|
1460
|
-
|
1461
|
-
# first clean and make... Should be compatible with any shell since
|
1462
|
-
# redirection is always wrapped in 'bash -c "{STUFF}"'
|
1463
|
-
simple_clean
|
1464
|
-
begin
|
1465
|
-
mk
|
1466
|
-
rescue TestCaseDirError
|
1467
|
-
return fail_test(:compilation)
|
1468
|
-
end
|
1469
|
-
|
1470
|
-
# remove old final model if it exists
|
1471
|
-
remove_final_model
|
1472
|
-
|
1473
|
-
# only check restart/photo if we get through run successfully
|
1474
|
-
check_restart if check_run
|
1475
|
-
|
1476
|
-
# get reported runtime, retries, backups, and steps
|
1477
|
-
load_summary_data if File.exist?(out_file)
|
1478
|
-
end
|
1479
|
-
|
1480
|
-
# append contents of err.txt to end of out.txt, then delete err.txt
|
1481
|
-
def append_and_rm_err(outfile = 'out.txt', errfile = 'err.txt')
|
1482
|
-
err_contents = File.read(errfile)
|
1483
|
-
display_errors(err_contents)
|
1484
|
-
log_errors(err_contents, outfile)
|
1485
|
-
FileUtils.rm errfile
|
1486
|
-
end
|
1487
|
-
|
1488
|
-
def display_errors(err_contents)
|
1489
|
-
return if err_contents.strip.empty?
|
1490
|
-
shell.say("\nERRORS", :red)
|
1491
|
-
puts err_contents
|
1492
|
-
shell.say('END OF ERRORS', :red)
|
1493
|
-
end
|
1494
|
-
|
1495
|
-
def log_errors(err_contents, outfile)
|
1496
|
-
return if err_contents.strip.empty?
|
1497
|
-
File.open(outfile, 'a') { |f_out| f_out.write(err_contents) }
|
1498
|
-
shell.say("appended to #{outfile}\n", :red)
|
1499
|
-
end
|
1500
|
-
|
1501
|
-
def simple_clean
|
1502
|
-
puts './clean'
|
1503
|
-
return if bash_execute('./clean')
|
1504
|
-
raise TestCaseDirError, 'Encountered an error when running `clean` in ' \
|
1505
|
-
"#{Dir.getwd} for test case #{test_name}."
|
1506
|
-
end
|
1507
|
-
|
1508
|
-
def mk
|
1509
|
-
puts './mk > mk.txt'
|
1510
|
-
unless bash_execute('./mk > mk.txt')
|
1511
|
-
raise TestCaseDirError, 'Encountered an error when running `mk` in ' \
|
1512
|
-
"#{Dir.getwd} for test case #{test_name}."
|
1513
|
-
end
|
1514
|
-
FileUtils.rm 'mk.txt'
|
1515
|
-
end
|
1516
|
-
|
1517
|
-
def remove_final_model
|
1518
|
-
# remove final model if it already exists
|
1519
|
-
return unless final_model
|
1520
|
-
return unless File.exist?(final_model)
|
1521
|
-
FileUtils.rm(final_model)
|
1522
|
-
end
|
1523
|
-
|
1524
|
-
def out_file
|
1525
|
-
File.join(test_case_dir, 'out.txt')
|
1526
|
-
end
|
1527
|
-
|
1528
|
-
# helpers for getting run summaries
|
1529
|
-
def run_summaries
|
1530
|
-
# look at all lines in out.txt
|
1531
|
-
lines = IO.readlines(out_file)
|
1532
|
-
|
1533
|
-
# find lines with summary information
|
1534
|
-
summary_line_numbers = []
|
1535
|
-
lines.each_with_index do |line, i|
|
1536
|
-
if line =~ /^\s*runtime \(minutes\),\s+retries,\s+backups,\ssteps/
|
1537
|
-
summary_line_numbers << i
|
1538
|
-
end
|
1539
|
-
end
|
1540
|
-
|
1541
|
-
# find lines indicating passage or failure of runs and restarts
|
1542
|
-
run_finish_line_numbers = []
|
1543
|
-
restart_finish_line_numbers = []
|
1544
|
-
lines.each_with_index do |line, i|
|
1545
|
-
if line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+restart/
|
1546
|
-
restart_finish_line_numbers << i
|
1547
|
-
elsif line =~ /^\s*((?:PASS)|(?:FAIL))\s+#{test_name}\s+run/
|
1548
|
-
run_finish_line_numbers << i
|
1549
|
-
end
|
1550
|
-
end
|
1551
|
-
|
1552
|
-
# only keep summaries that correspond to runs rather than restart
|
1553
|
-
summary_line_numbers.select do |i|
|
1554
|
-
run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
|
1555
|
-
end.map { |line_number| lines[line_number] }
|
1556
|
-
end
|
1557
|
-
|
1558
|
-
def get_summary_text
|
1559
|
-
# original plan was to include diff data in summary text... now it's just
|
1560
|
-
# part of the test_instance object and is submitted as an integer
|
1561
|
-
# res = case diff
|
1562
|
-
# when 0
|
1563
|
-
# "No Diff\n"
|
1564
|
-
# when 1
|
1565
|
-
# "Diff\n"
|
1566
|
-
# else
|
1567
|
-
# "Ambiguous Diff\n"
|
1568
|
-
# end
|
1569
|
-
# res +
|
1570
|
-
IO.readlines(out_file).select do |line|
|
1571
|
-
line =~ /^\s*runtime/
|
1572
|
-
end.join
|
1573
|
-
end
|
1574
|
-
|
1575
|
-
def run_summary?(i, run_finish_line_numbers, restart_finish_line_numbers)
|
1576
|
-
# iterate from starting line (a summary line) up to largest PASS/FAIL
|
1577
|
-
# line, bail out if summary line is beyond any PASS/FAIL line
|
1578
|
-
max_line = run_finish_line_numbers.max || 0
|
1579
|
-
max_line = [max_line, (restart_finish_line_numbers.max || 0)].max
|
1580
|
-
return false if i > max_line
|
1581
|
-
# return true if next PASS/FAIL line is for a run and fail if it is for a
|
1582
|
-
# restart
|
1583
|
-
i.upto(max_line) do |j|
|
1584
|
-
return true if run_finish_line_numbers.include?(j)
|
1585
|
-
return false if restart_finish_line_numbers.include?(j)
|
1586
|
-
end
|
1587
|
-
false
|
856
|
+
is_valid = dir_or_symlink_exists? test_suite_dir
|
857
|
+
raise MesaDirError, "Invalid MESA dir: #{mesa.mesa_dir}" unless is_valid
|
1588
858
|
end
|
1589
859
|
|
1590
860
|
end
|
@@ -1599,13 +869,11 @@ end
|
|
1599
869
|
def visit_and_check(new_dir, exception, message)
|
1600
870
|
cwd = Dir.getwd
|
1601
871
|
shell.say "Leaving #{cwd}", :blue
|
1602
|
-
|
1603
|
-
shell.say "Entering #{new_dir}.", :blue
|
872
|
+
shell.say "\nEntering #{new_dir}.", :blue
|
1604
873
|
Dir.chdir(new_dir)
|
1605
874
|
success = yield if block_given?
|
1606
875
|
shell.say "Leaving #{new_dir}", :blue
|
1607
|
-
|
1608
|
-
shell.say "Entering #{cwd}.", :blue
|
876
|
+
shell.say "\nEntering #{cwd}.", :blue
|
1609
877
|
Dir.chdir(cwd)
|
1610
878
|
return if success
|
1611
879
|
raise exception, message
|
@@ -1613,18 +881,18 @@ end
|
|
1613
881
|
|
1614
882
|
# cd into a new directory, execute a block, then cd back into original
|
1615
883
|
# directory
|
1616
|
-
def visit_dir(new_dir)
|
884
|
+
def visit_dir(new_dir, quiet: false)
|
1617
885
|
cwd = Dir.getwd
|
1618
|
-
shell.say "Leaving #{cwd}\n", :blue
|
1619
|
-
shell.say "
|
886
|
+
shell.say "Leaving #{cwd}\n", :blue unless quiet
|
887
|
+
shell.say "\nEntering #{new_dir}.", :blue unless quiet
|
1620
888
|
Dir.chdir(new_dir)
|
1621
889
|
yield if block_given?
|
1622
|
-
shell.say "Leaving #{new_dir}\n", :blue
|
1623
|
-
shell.say "
|
1624
|
-
puts ""
|
890
|
+
shell.say "Leaving #{new_dir}\n", :blue unless quiet
|
891
|
+
shell.say "\nRe-entering #{cwd}.", :blue unless quiet
|
1625
892
|
Dir.chdir(cwd)
|
1626
893
|
end
|
1627
894
|
|
895
|
+
# the next function probalby doesn't belong here, but keep it anyway, please
|
1628
896
|
# create seed data for test cases for MesaTestHub of a given mesa version
|
1629
897
|
def generate_seeds_rb(mesa_dir, outfile)
|
1630
898
|
m = Mesa.new(mesa_dir: mesa_dir)
|
@@ -1635,7 +903,6 @@ def generate_seeds_rb(mesa_dir, outfile)
|
|
1635
903
|
m.test_names.each do |test_case_name|
|
1636
904
|
f.puts ' {'
|
1637
905
|
f.puts " name: '#{test_case_name}',"
|
1638
|
-
f.puts " version_added: #{m.version_number},"
|
1639
906
|
# no comma on last one
|
1640
907
|
if test_case_name == m.test_names[-1]
|
1641
908
|
f.puts(' }')
|
@@ -1648,12 +915,22 @@ def generate_seeds_rb(mesa_dir, outfile)
|
|
1648
915
|
end
|
1649
916
|
end
|
1650
917
|
|
1651
|
-
# Check if path is directory or symlink
|
1652
918
|
def dir_or_symlink_exists?(path)
|
1653
|
-
|
919
|
+
File.directory?(path) || File.symlink?(path)
|
1654
920
|
end
|
1655
921
|
|
1656
922
|
# force the execution to happen with bash
|
1657
|
-
def bash_execute(command)
|
1658
|
-
system('bash -c "' + command + '"')
|
923
|
+
def bash_execute(command, throw_exception=false)
|
924
|
+
res = system('bash -c "' + command + '"')
|
925
|
+
if !res && throw_exception
|
926
|
+
raise BashError('Encountered an error when executing the following '\
|
927
|
+
"command in bash: #{command}.")
|
928
|
+
end
|
929
|
+
res
|
930
|
+
end
|
931
|
+
|
932
|
+
# force execution to happen with bash, but return result rather than exit
|
933
|
+
# status (like backticks)
|
934
|
+
def bashticks(command)
|
935
|
+
`bash -c "#{command}"`.chomp
|
1659
936
|
end
|