mesa_test 0.2.13 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/mesa_test +262 -432
- data/lib/mesa_test.rb +445 -1163
- metadata +7 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 56e31a9841e2aae7f051e81c87283288c30a760dbf86799f34e3c4106e05536a
|
4
|
+
data.tar.gz: caef468ad982b2d508c56067c3b10e9f645f0a02cccb86f4d770b94ce3e190a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '09e0a052e5628e28ed6786811e5861548c02924153fe74c133e6a1265f5bdaa1e35cd479bae5181c14423ac04b5588d28223d441c8959f56629ebfbbc6f33b9f'
|
7
|
+
data.tar.gz: bb40a700dc352a309191c90550d9ab598ad9d2a2f5d9e123590f1d89960962bcc54d1e32afa2e06916bf6cf886e8391b8a18f20efecd0fbcd68024dd755bad8b
|
data/bin/mesa_test
CHANGED
@@ -1,10 +1,13 @@
|
|
1
1
|
#! /usr/bin/env ruby
|
2
2
|
|
3
|
-
# if
|
4
|
-
# rather than system version
|
5
|
-
|
6
|
-
|
7
|
-
|
3
|
+
# if :development, submit cases to localhost:3000 and load local mesa_test library
|
4
|
+
# rather than system version and run out of local lib, not installed
|
5
|
+
# if :staging, submit to beta version and run out of local lib, not installed
|
6
|
+
# anything else: trust rubygems to get lib and submit to real deal testhub
|
7
|
+
MODE = :production
|
8
|
+
|
9
|
+
case MODE
|
10
|
+
when :staging, :development
|
8
11
|
require_relative '../lib/mesa_test'
|
9
12
|
else
|
10
13
|
require 'mesa_test'
|
@@ -15,448 +18,307 @@ require 'yaml'
|
|
15
18
|
|
16
19
|
class MesaTest < Thor
|
17
20
|
DEVELOPMENT_URI = 'http://localhost:3000'.freeze
|
21
|
+
STAGING_URI = 'https://beta-testhub.herokuapp.com'.freeze
|
18
22
|
|
19
|
-
desc '
|
23
|
+
desc 'test [TEST CASE]', 'run and submit a test case or commit'
|
20
24
|
long_desc <<-LONGDESC
|
21
|
-
If optional
|
22
|
-
|
23
|
-
|
24
|
-
the
|
25
|
+
If you specify an optional test case, only that test case
|
26
|
+
will be tested. Otherwise all test cases in your work directory (set up in
|
27
|
+
your .mesa_test.yml file). TEST_CASE can also be an integer corresponding
|
28
|
+
to the position it has in do1_test_source (or a concatenated version of
|
29
|
+
that file if no module is specified). Modules are searched/concatenated in
|
30
|
+
this order:
|
31
|
+
|
32
|
+
#{MesaTestCase.modules.map { |mod| ' ' + mod.to_s }.join(', ')}
|
25
33
|
LONGDESC
|
26
|
-
def setup(config_file = File.join(ENV['HOME'], '.mesa_test.yml'))
|
27
|
-
MesaTestSubmitter.new_from_config(config_file: config_file,
|
28
|
-
force_setup: true)
|
29
|
-
end
|
30
34
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
to the line number of the test name in its do1_test_source file (or a
|
36
|
-
concatenated version of that file if no module is specified). Modules are
|
37
|
-
searched/concatenated in this order:
|
38
|
-
|
39
|
-
#{MesaTestCase.modules.map { |mod| ' ' + mod.to_s }.join(', ')}
|
40
|
-
LONGDESC
|
41
|
-
option :force, desc: 'When submitting, assume submission data in ' +
|
42
|
-
'~/.mesa_test.yml are correct rather than checking ' +
|
43
|
-
'with user.', type: :boolean, default: false,
|
44
|
-
aliases: '-f'
|
35
|
+
option :force, desc: 'When submitting, assume submission data in ' \
|
36
|
+
'~/.mesa_test.yml are correct rather than checking ' \
|
37
|
+
'with user.',
|
38
|
+
type: :boolean, default: true, aliases: '-f'
|
45
39
|
option :submit, desc: 'Submit results (if any) to MesaTestHub.',
|
46
40
|
type: :boolean, default: true
|
47
|
-
option :diff, type: :boolean, default: false, desc: 'Assume checksums are ' +
|
48
|
-
'up to date and perform diff', aliases: '-d'
|
49
|
-
option :svn, desc: 'Use svn to gather log/version data. Safe to use when ' +
|
50
|
-
'also using --no-submit; else may give bad version numbers.',
|
51
|
-
type: :boolean, default: true
|
52
|
-
option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
|
53
|
-
' if the SDK is used and instead use value of --using_sdk.',
|
54
|
-
type: :boolean, default: false
|
55
|
-
option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
|
56
|
-
'determine SDK version. Ignored unless using --set-sdk).',
|
57
|
-
type: :boolean, default: true
|
58
41
|
option :module, desc: 'Which module to test and/or submit, if applicable.',
|
59
|
-
type: :string, default: 'all', aliases: '-m'
|
60
|
-
option :
|
61
|
-
|
62
|
-
option :auto_diff, desc: 'Automatically determine if diffs should be run '\
|
63
|
-
'from svn log entry. Overrides --diff or --no-diff.',
|
64
|
-
type: :boolean, default: false
|
65
|
-
def test_one(mesa_dir, test_case_name)
|
66
|
-
s = create_and_check_submitter(force: options[:force]) if options[:submit]
|
67
|
-
m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
|
68
|
-
mod: options[:module].downcase.to_sym,
|
69
|
-
use_svn: options[:svn], using_sdk: using_sdk?(s),
|
70
|
-
auto_diff: options[:auto_diff])
|
71
|
-
t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
|
72
|
-
mod: options[:module].downcase.to_sym)
|
73
|
-
|
74
|
-
# clean and run test
|
75
|
-
t.clean
|
76
|
-
t.do_one
|
77
|
-
|
78
|
-
# log results
|
79
|
-
t.log_results if options[:log]
|
80
|
-
|
81
|
-
# bail out if not submitting
|
82
|
-
return unless options[:submit]
|
83
|
-
|
84
|
-
# submit results
|
85
|
-
print 'Submitting results to ' + s.base_uri + '... '
|
86
|
-
s.submit(t)
|
87
|
-
puts "Done.\n"
|
88
|
-
end
|
42
|
+
type: :string, default: 'all', aliases: '-m'
|
43
|
+
option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
|
44
|
+
type: :string
|
89
45
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
46
|
+
def test(test_case_name = nil)
|
47
|
+
s = if options[:submit]
|
48
|
+
create_and_check_submitter(force: options[:force])
|
49
|
+
else
|
50
|
+
create_submitter(force: options[:force])
|
51
|
+
end
|
52
|
+
m = create_and_check_mesa(
|
53
|
+
mesa_dir: work_dir(options[:mesadir], s.mesa_work),
|
54
|
+
mirror_dir: s.mesa_mirror,
|
55
|
+
mod: options[:module].downcase.to_sym,
|
56
|
+
github_protocol: s.github_protocol
|
57
|
+
)
|
58
|
+
if test_case_name
|
59
|
+
# only testing one test case
|
60
|
+
t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
|
61
|
+
mod: options[:module].downcase.to_sym)
|
62
|
+
|
63
|
+
# run test
|
64
|
+
t.do_one
|
65
|
+
|
66
|
+
# bail out if not submitting
|
67
|
+
return unless options[:submit]
|
68
|
+
|
69
|
+
# submit results
|
70
|
+
shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
|
71
|
+
s.submit_instance(m, t)
|
72
|
+
else
|
73
|
+
# run all tests
|
74
|
+
m.each_test_run(mod: options[:module].downcase.to_sym)
|
98
75
|
|
99
|
-
|
100
|
-
|
101
|
-
'with user.', type: :boolean, default: false,
|
102
|
-
aliases: '-f'
|
103
|
-
option :submit, desc: 'Submit results (if any) to MesaTestHub.',
|
104
|
-
type: :boolean, default: true
|
105
|
-
option :diff, type: :boolean, default: false, desc: 'Assume checksums are ' +
|
106
|
-
'up to date and perform diff', aliases: '-d'
|
107
|
-
option :svn, desc: 'Use svn to gather log/version data. Safe to use when ' +
|
108
|
-
'also using --no-submit; else may give bad version numbers.',
|
109
|
-
type: :boolean, default: true
|
110
|
-
option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
|
111
|
-
' if the SDK is used and instead use value of --using_sdk.',
|
112
|
-
type: :boolean, default: false
|
113
|
-
option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
|
114
|
-
'determine SDK version. Ignored unless using --set-sdk).',
|
115
|
-
type: :boolean, default: true
|
116
|
-
option :module, desc: 'Which module to test and/or submit, if applicable.',
|
117
|
-
type: :string, default: 'all', aliases: '-m'
|
118
|
-
option :log, type: :boolean, default: true, desc: 'Save yml file of test ' \
|
119
|
-
'results in test case directory.'
|
120
|
-
option :auto_diff, desc: 'Automatically determine if diffs should be run '\
|
121
|
-
'from svn log entry. Overrides --diff or --no-diff.',
|
122
|
-
type: :boolean, default: false
|
123
|
-
def test_revision(mesa_dir)
|
124
|
-
s = create_and_check_submitter(force: options[:force]) if options[:submit]
|
125
|
-
m = create_and_check_mesa(mesa_dir: mesa_dir, diff: options[:diff],
|
126
|
-
mod: options[:module].downcase.to_sym,
|
127
|
-
use_svn: options[:svn],
|
128
|
-
using_sdk: using_sdk?(s),
|
129
|
-
auto_diff: options[:auto_diff])
|
130
|
-
|
131
|
-
# run all tests
|
132
|
-
m.each_test_run_and_diff(log_results: options[:log],
|
133
|
-
mod: options[:module].downcase.to_sym)
|
134
|
-
|
135
|
-
# submit all tests
|
136
|
-
s.submit_revision(m) if options[:submit]
|
137
|
-
end
|
76
|
+
# bail out if not submitting
|
77
|
+
return unless options[:submit]
|
138
78
|
|
79
|
+
shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
|
80
|
+
s.submit_commit(m)
|
81
|
+
end
|
82
|
+
shell.say "Done.\n", :green
|
83
|
+
end
|
139
84
|
|
140
|
-
desc '
|
85
|
+
desc 'submit [TEST CASE]', 'submit a test case or commit'
|
141
86
|
long_desc <<-LONGDESC
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
87
|
+
All tests are assumed to be in the default mesa directory (set as mesa_work
|
88
|
+
in .mesa_test.yml. If you specify an optional test case, only that test case
|
89
|
+
will be submitted. TEST_CASE can also be an integer corresponding to the
|
90
|
+
position it has in do1_test_source (or a concatenated version of that file
|
91
|
+
if no module is specified). Modules are searched/concatenated in this
|
92
|
+
order:
|
93
|
+
|
94
|
+
#{MesaTestCase.modules.map { |mod| ' ' + mod.to_s }.join(', ')}
|
148
95
|
LONGDESC
|
149
96
|
|
150
|
-
option :force, desc: 'When submitting, assume submission data in '
|
151
|
-
'~/.mesa_test.yml are correct rather than checking '
|
152
|
-
'with user.',
|
153
|
-
|
154
|
-
option :svn, desc: 'Use svn to gather log/version data. NECESSARY TO GET ' +
|
155
|
-
'ACCURATE VERSION NUMBER.', type: :boolean, default: true
|
156
|
-
option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
|
157
|
-
' if the SDK is used and instead use value of --using_sdk.',
|
158
|
-
type: :boolean, default: false
|
159
|
-
option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
|
160
|
-
'determine SDK version. Ignored unless using --set-sdk).',
|
161
|
-
type: :boolean, default: true
|
97
|
+
option :force, desc: 'When submitting, assume submission data in ' \
|
98
|
+
'~/.mesa_test.yml are correct rather than checking ' \
|
99
|
+
'with user.',
|
100
|
+
type: :boolean, default: true, aliases: '-f'
|
162
101
|
option :module, desc: 'Which module to test and/or submit, if applicable.',
|
163
102
|
type: :string, default: 'all', aliases: '-m'
|
164
|
-
|
103
|
+
option :empty, desc: 'Whether any test cases will be submitted. Useful for '\
|
104
|
+
'indicating compilation status independently of ' \
|
105
|
+
'single test submissions.',
|
106
|
+
type: :boolean, default: false, aliases: '-e'
|
107
|
+
option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
|
108
|
+
type: :string
|
109
|
+
|
110
|
+
def submit(test_case_name = nil)
|
165
111
|
s = create_and_check_submitter(force: options[:force])
|
166
|
-
m = create_and_check_mesa(
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
#
|
175
|
-
|
176
|
-
|
177
|
-
|
112
|
+
m = create_and_check_mesa(
|
113
|
+
mesa_dir: work_dir(options[:mesadir], s.mesa_work),
|
114
|
+
mirror_dir: s.mesa_mirror,
|
115
|
+
mod: options[:module].downcase.to_sym,
|
116
|
+
github_protocol: s.github_protocol
|
117
|
+
)
|
118
|
+
empty = options[:empty]
|
119
|
+
|
120
|
+
# default value for now; might change to false if we have a bad
|
121
|
+
# installation or if a test case is specified
|
122
|
+
|
123
|
+
if test_case_name
|
124
|
+
# just submitting one test case.
|
125
|
+
# this is not an empty submission since there is one test case (the
|
126
|
+
# +empty+ option is ignored in this case)
|
127
|
+
|
128
|
+
t = create_and_check_test_case(mesa: m, test_case_name: test_case_name,
|
129
|
+
mod: options[:module].downcase.to_sym)
|
130
|
+
# submit results
|
131
|
+
shell.say 'Submitting results to ' + s.base_uri + '... ', :blue
|
132
|
+
s.submit_instance(m, t)
|
133
|
+
shell.say "Done.\n", :green
|
134
|
+
else
|
135
|
+
# submitting compilation information and either all tests (entire) or
|
136
|
+
# no tests (empty)
|
137
|
+
begin
|
138
|
+
m.check_installation
|
139
|
+
rescue MesaDirError
|
140
|
+
shell.say %q{This MESA installation doesn't seem to be compiled } \
|
141
|
+
'properly. Submitting a compilation failure to MESATestHub.', :red
|
142
|
+
empty = true
|
143
|
+
ensure
|
144
|
+
# submit all tests
|
145
|
+
s.submit_commit(m, empty: empty)
|
146
|
+
end
|
147
|
+
end
|
178
148
|
end
|
179
149
|
|
180
|
-
desc '
|
150
|
+
desc 'checkout [SHA]', 'checkout a commit'
|
181
151
|
long_desc <<-LONGDESC
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
152
|
+
Checks out a particular commit to default mesa directory (mesa_work in
|
153
|
+
.mesa_test.yml). Uses git worktree magic to avoid re-downloading all of
|
154
|
+
mesa history if a mirror repo is already set up. Does not affect the
|
155
|
+
user's MESA_DIR or other environment variables.
|
156
|
+
|
157
|
+
If no sha is provided, defaults to main (if checkout is occuring).
|
188
158
|
LONGDESC
|
189
|
-
option :force, desc: 'When submitting, assume submission data in ' +
|
190
|
-
'~/.mesa_test.yml are correct rather than checking ' +
|
191
|
-
'with user.', type: :boolean, default: false,
|
192
|
-
aliases: '-f'
|
193
|
-
option :module, desc: 'Which module to test and/or submit, if applicable.',
|
194
|
-
type: :string, default: 'all', aliases: '-m'
|
195
|
-
option :svn, desc: 'Use svn to gather log/version data. NECESSARY TO GET ' +
|
196
|
-
'ACCURATE VERSION NUMBER.', type: :boolean, default: true
|
197
|
-
option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
|
198
|
-
' if the SDK is used and instead use value of --using_sdk.',
|
199
|
-
type: :boolean, default: false
|
200
|
-
option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
|
201
|
-
'determine SDK version. Ignored unless using --set-sdk).',
|
202
|
-
type: :boolean, default: true
|
203
|
-
def submit_revision(mesa_dir)
|
204
|
-
s = create_and_check_submitter(force: options[:force])
|
205
|
-
m = create_and_check_mesa(mesa_dir: mesa_dir,
|
206
|
-
mod: options[:module].downcase.to_sym,
|
207
|
-
use_svn: options[:svn], using_sdk: using_sdk?(s))
|
208
|
-
begin
|
209
|
-
m.check_installation
|
210
|
-
rescue MesaDirError
|
211
|
-
shell.say %q{This MESA installation doesn't seem to be compiled } \
|
212
|
-
'properly. Submitting a compilation failure to MESATestHub.', :red
|
213
|
-
else
|
214
|
-
# load all test data
|
215
|
-
m.each_test_load_results(mod: options[:module].downcase.to_sym)
|
216
|
-
ensure
|
217
|
-
# submit all tests
|
218
|
-
s.submit_revision(m)
|
219
|
-
end
|
220
|
-
end
|
221
159
|
|
160
|
+
option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
|
161
|
+
type: :string
|
162
|
+
|
163
|
+
def checkout(sha = 'main')
|
164
|
+
# assuming config wizard has already been set (perhaps a little reckless)
|
165
|
+
# so that we can get at the work directory and mirror directory
|
166
|
+
s = create_submitter(force: true)
|
167
|
+
Mesa.checkout(sha: sha,
|
168
|
+
work_dir: work_dir(options[:mesadir], s.mesa_work),
|
169
|
+
mirror_dir: s.mesa_mirror,
|
170
|
+
github_protocol: s.github_protocol)
|
171
|
+
end
|
222
172
|
|
223
|
-
desc 'install
|
224
|
-
'VERSION_NUMBER to directory MESA_DIR'
|
173
|
+
desc 'install [SHA]', 'checkout and install a commit'
|
225
174
|
long_desc <<-LONGDESC
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
175
|
+
Optionally checks out a particular commit to default mesa directory
|
176
|
+
(mesa_work in .mesa_test.yml). Then runs installation script in the work
|
177
|
+
directory. Equivalent to an [optional] git checkout followed by going
|
178
|
+
into the directory and running ./clean and ./install, but there is some git
|
179
|
+
worktree magic on the backend to improve efficiency. SDK or compilers must be
|
180
|
+
set up prior. Does not affect the user's MESA_DIR or other environment
|
181
|
+
variables outside of execution.
|
182
|
+
|
183
|
+
If no sha is provided, defaults to main (if checkout is occuring). SHA is
|
184
|
+
ignored if using option --no-checkout
|
230
185
|
LONGDESC
|
231
|
-
|
232
|
-
|
186
|
+
|
187
|
+
option :checkout, desc: 'Whether a checkout needs to happen. Use '\
|
188
|
+
'--no-checkout if you already ran checkout.',
|
189
|
+
type: :boolean, default: true, aliases: '-c'
|
190
|
+
option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
|
191
|
+
type: :string
|
192
|
+
|
193
|
+
def install(sha = 'main')
|
194
|
+
# assuming config wizard has already been set (perhaps a little reckless)
|
195
|
+
# so that we can get at the work directory and mirror directory
|
196
|
+
s = create_submitter(force: true)
|
197
|
+
m = if options[:checkout]
|
198
|
+
Mesa.checkout(
|
199
|
+
sha: sha,
|
200
|
+
work_dir: work_dir(options[:mesadir], s.mesa_work),
|
201
|
+
mirror_dir: s.mesa_mirror,
|
202
|
+
github_protocol: s.github_protocol
|
203
|
+
)
|
204
|
+
else
|
205
|
+
create_and_check_mesa(
|
206
|
+
mesa_dir: work_dir(options[:mesadir], s.mesa_work),
|
207
|
+
mirror_dir: s.mesa_mirror,
|
208
|
+
mod: :all,
|
209
|
+
github_protocol: s.github_protocol
|
210
|
+
)
|
211
|
+
end
|
233
212
|
m.clean
|
234
213
|
m.install
|
235
214
|
end
|
236
215
|
|
237
|
-
desc '
|
238
|
-
'install mesa release VERSION_NUMBER to directory MESA_DIR, run all tests,'\
|
239
|
-
'and submit version and test data in single request.'
|
216
|
+
desc 'install_and_test [SHA]', 'Install, test, and submit an entire commit.'
|
240
217
|
long_desc <<-LONGDESC
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
218
|
+
Optionally checks out a particular commit (specified by SHA, which default to
|
219
|
+
HEAD if not provided) to a mesa directory (mesa_work in .mesa_test.yml,
|
220
|
+
unless a custom location is provided by --workdir=MYDIR). Then go into the
|
221
|
+
directory and run ./clean and ./install. SDK or compilers must be set up
|
222
|
+
prior. Once installation is complete, run the test suite, and report results
|
223
|
+
(commit and all test data) to MesaTestHub in a single request.
|
247
224
|
Does not affect the user's MESA_DIR or other environment variables. This is
|
248
225
|
basically a shortcut for running
|
249
226
|
|
250
|
-
`mesa_test install
|
227
|
+
`mesa_test install SOME_SHA`
|
251
228
|
|
252
229
|
followed by
|
253
230
|
|
254
|
-
`mesa_test
|
231
|
+
`mesa_test test`
|
232
|
+
|
233
|
+
If no sha is provided, defaults to main (if checkout is occuring). SHA is
|
234
|
+
ignored if using option --no-checkout
|
255
235
|
|
256
|
-
If VERSION_NUMBER is 'non-paxton' or 'non_paxton', then the most recent
|
257
|
-
commit by anyone other than Bill Paxton ('bill_paxton') is tested and the
|
258
|
-
--diff flag is ignored and set to false. Note that using 'HEAD' for version
|
259
|
-
number will automatically select the most recent release, regardless of the
|
260
|
-
author.
|
261
236
|
LONGDESC
|
262
237
|
|
263
|
-
option :
|
264
|
-
|
265
|
-
|
266
|
-
|
238
|
+
option :checkout, desc: 'Whether a checkout needs to happen. Use '\
|
239
|
+
'--no-checkout if you already ran checkout.',
|
240
|
+
type: :boolean, default: true, aliases: '-c'
|
241
|
+
option :mesadir, desc: 'Location of mesa dir, if not relying on config data',
|
242
|
+
type: :string
|
243
|
+
option :force, desc: 'When submitting, assume submission data in ' \
|
244
|
+
'~/.mesa_test.yml are correct rather than checking ' \
|
245
|
+
'with user.',
|
246
|
+
type: :boolean, default: true, aliases: '-f'
|
267
247
|
option :submit, desc: 'Submit results (if any) to MesaTestHub.',
|
268
248
|
type: :boolean, default: true
|
269
|
-
option :diff, type: :boolean, default: false, desc: 'Assume checksums are ' +
|
270
|
-
'up to date and perform diff', aliases: '-d'
|
271
|
-
option :svn, desc: 'Use svn to gather log/version data. Safe to use when ' +
|
272
|
-
'also using --no-submit; else may give bad version numbers.',
|
273
|
-
type: :boolean, default: true
|
274
|
-
option :set_sdk, desc: 'Override user- and test-generated data to deterine' +
|
275
|
-
' if the SDK is used and instead use value of --using_sdk.',
|
276
|
-
type: :boolean, default: false
|
277
|
-
option :using_sdk, desc: 'Assume we are using the SDK and automatically ' +
|
278
|
-
'determine SDK version. Ignored unless using --set-sdk).',
|
279
|
-
type: :boolean, default: true
|
280
249
|
option :module, desc: 'Which module to test and/or submit, if applicable.',
|
281
250
|
type: :string, default: 'all', aliases: '-m'
|
282
251
|
option :destroy, type: :boolean, desc: 'Delete mesa installation upon ' \
|
283
252
|
'submission of complete results.', default: false
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
253
|
+
|
254
|
+
def install_and_test(sha='main')
|
255
|
+
s = if options[:submit]
|
256
|
+
create_and_check_submitter(force: options[:force])
|
257
|
+
else
|
258
|
+
# not submitting? don't check config file (risky!) still need
|
259
|
+
# submitter to get work and mirror directories, though
|
260
|
+
create_submitter(force: true)
|
261
|
+
end
|
262
|
+
|
263
|
+
m = if options[:checkout]
|
264
|
+
Mesa.checkout(
|
265
|
+
sha: sha,
|
266
|
+
work_dir: work_dir(options[:mesadir], s.mesa_work),
|
267
|
+
mirror_dir: s.mesa_mirror,
|
268
|
+
github_protocol: s.github_protocol
|
269
|
+
)
|
270
|
+
else
|
271
|
+
create_and_check_mesa(
|
272
|
+
mesa_dir: work_dir(options[:mesadir], s.mesa_work),
|
273
|
+
mirror_dir: s.mesa_mirror,
|
274
|
+
mod: options[:module].to_sym,
|
275
|
+
github_protocol: s.github_protocol
|
276
|
+
)
|
277
|
+
end
|
278
|
+
|
300
279
|
begin
|
301
280
|
m.clean
|
302
281
|
m.install
|
303
282
|
rescue MesaDirError
|
304
|
-
shell.say "\nFailed in compiling MESA
|
283
|
+
shell.say "\nFailed in compiling MESA.", :red
|
305
284
|
else
|
306
285
|
check_mesa(m)
|
307
|
-
update_mesa(mesa: m,
|
308
|
-
auto_diff: options[:auto_diff])
|
286
|
+
update_mesa(mesa: m, mod: options[:module].downcase.to_sym)
|
309
287
|
|
310
288
|
# run all tests. Don't be fooled, whether or not a diff happens is set
|
311
289
|
# by the diff attribute of the mesa object, set above.
|
312
|
-
m.
|
313
|
-
log_results: options[:log])
|
290
|
+
m.each_test_run(mod: options[:module].downcase.to_sym)
|
314
291
|
ensure
|
315
|
-
# submit all
|
292
|
+
# submit all commit and test data
|
316
293
|
successfully_submitted = false
|
317
|
-
successfully_submitted = s.
|
294
|
+
successfully_submitted = s.submit_commit(m) if options[:submit]
|
318
295
|
|
319
296
|
# if requested, installation and submission are successful, destroy the
|
320
297
|
# directory
|
321
|
-
m.
|
322
|
-
end
|
323
|
-
end
|
324
|
-
|
325
|
-
desc 'search "SEARCH_QUERY" [EMAIL] [PASSWORD]', 'retrieve JSON array of ' \
|
326
|
-
'test instances matching SEARCH_QUERY'
|
327
|
-
long_desc <<-LONGDESC
|
328
|
-
Sends a JSON request to the Mesa TestHub API. All of your SEARCH_QUERY should
|
329
|
-
be in quotes. For details on how to form a proper search query, visit
|
330
|
-
testhub.mesastar.org/test_instances/search, where you can also test out
|
331
|
-
the search capability in a browser interface. Result is a single line of
|
332
|
-
JSON printed to STDOUT.
|
333
|
-
|
334
|
-
To prevent abuse, the API requires authentication with email and password.
|
335
|
-
By default, these are pulled from ~/.mesa_test.yml, but you may override
|
336
|
-
these with optional arguments.
|
337
|
-
LONGDESC
|
338
|
-
|
339
|
-
def search(search_query, email=nil, password=nil)
|
340
|
-
# get/confirm email and password
|
341
|
-
yml_file = File.join(ENV['HOME'], '.mesa_test.yml')
|
342
|
-
email, password = get_email_and_password(yml_file, email, password)
|
343
|
-
# prep API request
|
344
|
-
base_uri = if DEVELOPMENT_MODE
|
345
|
-
'http://localhost:3000'
|
346
|
-
else
|
347
|
-
'https://testhub.mesastar.org'
|
348
|
-
end
|
349
|
-
uri_search = URI.parse(base_uri + '/test_instances/search.json')
|
350
|
-
params = {
|
351
|
-
email: email,
|
352
|
-
password: password,
|
353
|
-
query_text: search_query
|
354
|
-
}
|
355
|
-
uri_search.query = URI.encode_www_form(params)
|
356
|
-
|
357
|
-
# perform search query
|
358
|
-
search_response = nil
|
359
|
-
Net::HTTP.start(uri_search.host, uri_search.port, use_ssl: base_uri.include?('https')) do |https|
|
360
|
-
request = Net::HTTP::Get.new(
|
361
|
-
uri_search,
|
362
|
-
initheader = { 'Content-Type' => 'application/json' }
|
363
|
-
)
|
364
|
-
search_response = JSON.parse(https.request(request).body)
|
365
|
-
end
|
366
|
-
|
367
|
-
# parse results
|
368
|
-
if search_response['error']
|
369
|
-
STDERR.puts "Error: #{search_response['error']}"
|
370
|
-
else
|
371
|
-
# break into sections
|
372
|
-
results = search_response["results"]
|
373
|
-
failures = search_response["failures"]
|
374
|
-
|
375
|
-
unless failures.empty?
|
376
|
-
STDERR.puts "The folloiwng options were invalid and ignored:"
|
377
|
-
failures.each { |failure| STDERR.puts "- #{failure}"}
|
378
|
-
end
|
379
|
-
|
380
|
-
shell.say results.to_json
|
298
|
+
m.remove if successfully_submitted && options[:destroy] && m.installed?
|
381
299
|
end
|
382
300
|
end
|
383
301
|
|
384
|
-
desc '
|
385
|
-
'instances matching SEARCH_QUERY'
|
302
|
+
desc 'setup [CONFIG_FILE]', 'Setup MesaTestHub config file.'
|
386
303
|
long_desc <<-LONGDESC
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
392
|
-
To prevent abuse, the API requires authentication with email and password.
|
393
|
-
By default, these are pulled from ~/.mesa_test.yml, but you may override
|
394
|
-
these with optional arguments.
|
304
|
+
If optional CONFIG_FILE is provided, search for that file and load it
|
305
|
+
to provide default values for configuration. If not provided, the default
|
306
|
+
file, ~/.mesa_test.yml, is used. The resulting config file is saved to
|
307
|
+
the same location it was read from.
|
395
308
|
LONGDESC
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
yml_file = File.join(ENV['HOME'], '.mesa_test.yml')
|
400
|
-
email, password = get_email_and_password(yml_file, email, password)
|
401
|
-
# prep API request
|
402
|
-
base_uri = if DEVELOPMENT_MODE
|
403
|
-
'http://localhost:3000'
|
404
|
-
else
|
405
|
-
'https://testhub.mesastar.org'
|
406
|
-
end
|
407
|
-
uri_count = URI.parse(base_uri + '/test_instances/search_count.json')
|
408
|
-
params = {
|
409
|
-
email: email,
|
410
|
-
password: password,
|
411
|
-
query_text: search_query
|
412
|
-
}
|
413
|
-
uri_count.query = URI.encode_www_form(params)
|
414
|
-
|
415
|
-
# perform count query
|
416
|
-
count_response = nil
|
417
|
-
Net::HTTP.start(uri_count.host, uri_count.port, use_ssl: base_uri.include?('https')) do |https|
|
418
|
-
request = Net::HTTP::Get.new(
|
419
|
-
uri_count,
|
420
|
-
initheader = { 'Content-Type' => 'application/json' }
|
421
|
-
)
|
422
|
-
count_response = JSON.parse(https.request(request).body)
|
423
|
-
end
|
424
|
-
|
425
|
-
# parse results
|
426
|
-
if count_response['error']
|
427
|
-
STDERR.puts "Error: #{count_response['error']}"
|
428
|
-
else
|
429
|
-
# break into sections
|
430
|
-
count = count_response["count"]
|
431
|
-
failures = count_response["failures"]
|
432
|
-
|
433
|
-
unless failures.empty?
|
434
|
-
STDERR.puts "The folloiwng options were invalid and ignored:"
|
435
|
-
failures.each { |failure| STDERR.puts "- #{failure}"}
|
436
|
-
end
|
437
|
-
|
438
|
-
shell.say count, :blue
|
439
|
-
end
|
309
|
+
def setup(config_file = File.join(ENV['HOME'], '.mesa_test', 'config.yml'))
|
310
|
+
MesaTestSubmitter.new_from_config(config_file: config_file,
|
311
|
+
force_setup: true)
|
440
312
|
end
|
441
313
|
|
442
314
|
private
|
443
315
|
|
444
|
-
def
|
445
|
-
if
|
446
|
-
|
316
|
+
def work_dir(given, default)
|
317
|
+
if given
|
318
|
+
File.expand_path('', given)
|
319
|
+
else
|
320
|
+
default
|
447
321
|
end
|
448
|
-
raise("File does not exist: #{file}.") unless File.exist?(file)
|
449
|
-
contents = File.read(file)
|
450
|
-
email_matcher = /^email\:\s*(?<email>.+@.+\..+)$/
|
451
|
-
password_matcher = /^password\:\s*(?<password>.+)$/
|
452
|
-
|
453
|
-
m = email_matcher.match(contents)
|
454
|
-
email ||= m[:email] if m
|
455
|
-
|
456
|
-
m = password_matcher.match(contents)
|
457
|
-
password ||= m[:password] if m
|
458
|
-
|
459
|
-
return [email, password]
|
460
322
|
end
|
461
323
|
|
462
324
|
def check_user_and_computer(submitter)
|
@@ -469,46 +331,52 @@ class MesaTest < Thor
|
|
469
331
|
end
|
470
332
|
end
|
471
333
|
|
472
|
-
def
|
473
|
-
s =
|
334
|
+
def create_submitter(force: true)
|
335
|
+
s = case MODE
|
336
|
+
when :development
|
474
337
|
MesaTestSubmitter.new_from_config(base_uri: DEVELOPMENT_URI)
|
338
|
+
when :staging
|
339
|
+
MesaTestSubmitter.new_from_config(base_uri: STAGING_URI)
|
475
340
|
else
|
476
341
|
MesaTestSubmitter.new_from_config
|
477
342
|
end
|
478
343
|
unless force
|
479
344
|
s.setup unless s.confirm_computer_data
|
480
345
|
end
|
346
|
+
s
|
347
|
+
end
|
348
|
+
|
349
|
+
def create_and_check_submitter(force: true)
|
350
|
+
s = create_submitter(force: force)
|
481
351
|
check_user_and_computer s
|
482
352
|
s
|
483
353
|
end
|
484
354
|
|
485
|
-
def create_and_check_mesa(mesa_dir
|
486
|
-
|
487
|
-
m = Mesa.new(mesa_dir: mesa_dir,
|
355
|
+
def create_and_check_mesa(mesa_dir: '', mirror_dir: '', mod: :all,
|
356
|
+
github_protocol: nil)
|
357
|
+
m = Mesa.new(mesa_dir: mesa_dir, mirror_dir: mirror_dir,
|
358
|
+
github_protocol: github_protocol)
|
488
359
|
check_mesa(m)
|
489
|
-
update_mesa(mesa: m,
|
490
|
-
m
|
360
|
+
update_mesa(mesa: m, mod: mod)
|
491
361
|
end
|
492
362
|
|
493
363
|
def check_mesa(mesa)
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
364
|
+
return if mesa.downloaded?
|
365
|
+
|
366
|
+
raise MesaDirError, "Invalid MESA_DIR: #{mesa.mesa_dir}. Please "\
|
367
|
+
'download and install a valid MESA version or provide the path to one.'
|
498
368
|
end
|
499
369
|
|
500
|
-
def update_mesa(mesa:,
|
370
|
+
def update_mesa(mesa:, mod: :all)
|
501
371
|
mesa.load_test_source_data(mod: mod)
|
502
|
-
|
503
|
-
mesa.update_checksums = !diff unless diff.nil?
|
504
|
-
# automatically figure out diff situation. Overrides choice from diff!
|
505
|
-
mesa.determine_diff if auto_diff
|
372
|
+
mesa
|
506
373
|
end
|
507
374
|
|
508
375
|
def create_and_check_test_case(mesa:, test_case_name:, mod: :all)
|
509
376
|
t = mesa.find_test_case(test_case_name: test_case_name, mod: mod)
|
510
377
|
# return test case if it was found
|
511
378
|
return t unless t.nil?
|
379
|
+
|
512
380
|
# test case was not found. Throw an error.
|
513
381
|
msg = "No such test case, #{test_case_name} found in any of "
|
514
382
|
# this should care about the value in `mod`. Current behavior is only
|
@@ -519,44 +387,6 @@ class MesaTest < Thor
|
|
519
387
|
msg << '.'
|
520
388
|
raise TestCaseDirError, msg
|
521
389
|
end
|
522
|
-
|
523
|
-
def filter_version_and_diff(version, submitter)
|
524
|
-
if %w[non-paxton non_paxton].include?(version.to_s.downcase)
|
525
|
-
[Mesa.last_non_paxton_revision(submitter.last_tested), false]
|
526
|
-
else
|
527
|
-
[version, options[:diff]]
|
528
|
-
end
|
529
|
-
end
|
530
|
-
|
531
|
-
# determine whether or not we can assume the test should be / was run with
|
532
|
-
# the SDK. This results should only be relevant if a test is GOING TO BE RUN.
|
533
|
-
# If it has already been run, then the compiler and compiler version have
|
534
|
-
# already been set (or the opportunity was lost, either way this won't help.)
|
535
|
-
#
|
536
|
-
# This gets used in a lot of places where it probably isn't necessary, just
|
537
|
-
# to prevent problems later on if things change, as it does no harm when it
|
538
|
-
# doesn't help.
|
539
|
-
#
|
540
|
-
# The only parameter is a submitter, if there is one to lean on. If there
|
541
|
-
# isn't, it's no problem.
|
542
|
-
def using_sdk?(s = nil)
|
543
|
-
# did user specify so? Trust them
|
544
|
-
if options[:set_sdk]
|
545
|
-
options[:using_sdk]
|
546
|
-
# are we submitting? scrape compiler info from that
|
547
|
-
elsif s
|
548
|
-
s.compiler == 'SDK'
|
549
|
-
# we don't have a submitter... need to read config file (same thing
|
550
|
-
# submitter is doing)
|
551
|
-
elsif File.exist?(File.join(ENV['HOME'], '.mesa_test.yml'))
|
552
|
-
data = YAML.load(File.read(File.join(ENV['HOME'], '.mesa_test.yml')))
|
553
|
-
data['compiler'] == 'SDK'
|
554
|
-
# nothing works. Don't tell mesa that we are using SDK (we could still be
|
555
|
-
# using it, but it's up to a future submitter to tell the web site so.)
|
556
|
-
else
|
557
|
-
false
|
558
|
-
end
|
559
|
-
end
|
560
390
|
end
|
561
391
|
|
562
392
|
# actually start the CLI
|