simp-beaker-helpers 1.34.3 → 1.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/pr_acceptance.yml +1 -1
- data/.github/workflows/pr_tests.yml +6 -7
- data/.rubocop.yml +652 -495
- data/CHANGELOG.md +4 -0
- data/Gemfile +5 -2
- data/lib/simp/beaker_helpers/constants.rb +7 -5
- data/lib/simp/beaker_helpers/inspec.rb +52 -55
- data/lib/simp/beaker_helpers/snapshot.rb +126 -134
- data/lib/simp/beaker_helpers/ssg.rb +33 -34
- data/lib/simp/beaker_helpers/version.rb +2 -1
- data/lib/simp/beaker_helpers/windows.rb +4 -1
- data/lib/simp/beaker_helpers.rb +274 -291
- data/lib/simp/rake/beaker.rb +174 -177
- data/spec/acceptance/suites/default/check_puppet_version_spec.rb +3 -3
- data/spec/acceptance/suites/default/fixture_modules_spec.rb +9 -9
- data/spec/acceptance/suites/default/install_simp_deps_repo_spec.rb +7 -13
- data/spec/acceptance/suites/default/pki_tests_spec.rb +10 -16
- data/spec/acceptance/suites/fips_from_fixtures/00_default_spec.rb +4 -4
- data/spec/acceptance/suites/inspec/00_default_spec.rb +22 -22
- data/spec/acceptance/suites/offline/00_default_spec.rb +43 -12
- data/spec/acceptance/suites/offline/nodesets/default.yml +1 -3
- data/spec/acceptance/suites/puppet_collections/00_default_spec.rb +3 -3
- data/spec/acceptance/suites/snapshot/00_snapshot_test_spec.rb +27 -7
- data/spec/acceptance/suites/snapshot/10_general_usage_spec.rb +3 -3
- data/spec/acceptance/suites/ssg/00_default_spec.rb +20 -18
- data/spec/acceptance/suites/windows/00_default_spec.rb +47 -49
- data/spec/acceptance/suites/windows/nodesets/default.yml +3 -3
- data/spec/acceptance/suites/windows/nodesets/win2012.yml +3 -3
- data/spec/acceptance/suites/windows/nodesets/win2016.yml +3 -3
- data/spec/acceptance/suites/windows/nodesets/win2019.yml +3 -3
- data/spec/lib/simp/beaker_helpers_spec.rb +96 -66
- data/spec/spec_helper.rb +51 -53
- data/spec/spec_helper_acceptance.rb +17 -22
- metadata +5 -5
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -40,8 +40,11 @@ group :system_tests do
|
|
40
40
|
gem 'beaker-windows'
|
41
41
|
gem 'ed25519'
|
42
42
|
gem 'net-ssh'
|
43
|
+
gem 'pry-byebug', '~> 3.10.0'
|
43
44
|
gem 'puppet', ENV.fetch('PUPPET_VERSION', ['>= 7.0.0', '< 9.0.0'])
|
44
45
|
gem 'puppetlabs_spec_helper', '>= 4.0.0', '< 8.0.0'
|
45
|
-
gem 'rubocop'
|
46
|
-
gem 'rubocop-
|
46
|
+
gem 'rubocop', '~> 1.69.2'
|
47
|
+
gem 'rubocop-performance', '~> 1.23.0'
|
48
|
+
gem 'rubocop-rake', '~> 0.6.0'
|
49
|
+
gem 'rubocop-rspec', '~> 3.3.0'
|
47
50
|
end
|
@@ -1,17 +1,19 @@
|
|
1
|
+
# SIMP namespace
|
1
2
|
module Simp; end
|
2
3
|
|
4
|
+
# SIMP Beaker helper methods for testing
|
3
5
|
module Simp::BeakerHelpers
|
4
6
|
# This is the *oldest* puppet-agent version that the latest release of SIMP supports
|
5
7
|
#
|
6
8
|
# This is done so that we know if some new thing that we're using breaks the
|
7
9
|
# oldest system that we support
|
8
|
-
DEFAULT_PUPPET_AGENT_VERSION = '~> 8.0'
|
10
|
+
DEFAULT_PUPPET_AGENT_VERSION = '~> 8.0'.freeze
|
9
11
|
|
10
12
|
SSG_REPO_URL = ENV['BEAKER_ssg_repo'] || 'https://github.com/ComplianceAsCode/content.git'
|
11
13
|
|
12
|
-
if ['true','yes'].include?(ENV['BEAKER_online'])
|
14
|
+
if ['true', 'yes'].include?(ENV['BEAKER_online'])
|
13
15
|
ONLINE = true
|
14
|
-
elsif ['false','no'].include?(ENV['BEAKER_online'])
|
16
|
+
elsif ['false', 'no'].include?(ENV['BEAKER_online'])
|
15
17
|
ONLINE = false
|
16
18
|
else
|
17
19
|
require 'open-uri'
|
@@ -19,8 +21,8 @@ module Simp::BeakerHelpers
|
|
19
21
|
begin
|
20
22
|
if URI.respond_to?(:open)
|
21
23
|
ONLINE = true if URI.open('http://google.com')
|
22
|
-
|
23
|
-
ONLINE = true
|
24
|
+
elsif open('http://google.com')
|
25
|
+
ONLINE = true
|
24
26
|
end
|
25
27
|
rescue
|
26
28
|
ONLINE = false
|
@@ -1,9 +1,9 @@
|
|
1
|
+
# SIMP Beaker helper methods for testing
|
1
2
|
module Simp::BeakerHelpers
|
2
3
|
require 'simp/beaker_helpers/constants'
|
3
4
|
|
4
5
|
# Helpers for working with Inspec
|
5
6
|
class Inspec
|
6
|
-
|
7
7
|
require 'json'
|
8
8
|
|
9
9
|
attr_reader :profile
|
@@ -12,16 +12,16 @@ module Simp::BeakerHelpers
|
|
12
12
|
|
13
13
|
def self.enable_repo_on(suts)
|
14
14
|
parallel = (ENV['BEAKER_SIMP_parallel'] == 'yes')
|
15
|
-
block_on(suts, :
|
15
|
+
block_on(suts, run_in_parallel: parallel) do |sut|
|
16
16
|
repo_manifest = create_yum_resource(
|
17
17
|
'chef-current',
|
18
18
|
{
|
19
|
-
:
|
20
|
-
:
|
21
|
-
}
|
19
|
+
baseurl: "https://packages.chef.io/repos/yum/current/el/#{fact_on(sut, 'os.release.major')}/$basearch",
|
20
|
+
gpgkeys: ['https://packages.chef.io/chef.asc']
|
21
|
+
},
|
22
22
|
)
|
23
23
|
|
24
|
-
apply_manifest_on(sut, repo_manifest, :
|
24
|
+
apply_manifest_on(sut, repo_manifest, catch_failures: true)
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
@@ -85,13 +85,13 @@ module Simp::BeakerHelpers
|
|
85
85
|
inspec_version = Gem::Version.new(on(@sut, 'inspec --version').output.lines.first.strip)
|
86
86
|
|
87
87
|
# See: https://github.com/inspec/inspec/pull/3935
|
88
|
-
if inspec_version <= Gem::Version.new('3.9.0')
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
88
|
+
inspec_cmd = if inspec_version <= Gem::Version.new('3.9.0')
|
89
|
+
"inspec exec '#{@test_dir}' --reporter json > #{sut_inspec_results}"
|
90
|
+
else
|
91
|
+
"inspec exec '#{@test_dir}' --chef-license accept --reporter json > #{sut_inspec_results}"
|
92
|
+
end
|
93
93
|
|
94
|
-
result = on(@sut, inspec_cmd, :
|
94
|
+
result = on(@sut, inspec_cmd, accept_all_exit_codes: true)
|
95
95
|
|
96
96
|
tmpdir = Dir.mktmpdir
|
97
97
|
begin
|
@@ -110,7 +110,7 @@ module Simp::BeakerHelpers
|
|
110
110
|
line.start_with?('{') && line.end_with?('}')
|
111
111
|
end
|
112
112
|
|
113
|
-
@results = JSON.
|
113
|
+
@results = JSON.parse(inspec_json) if inspec_json
|
114
114
|
rescue JSON::ParserError, JSON::GeneratorError
|
115
115
|
@results = nil
|
116
116
|
end
|
@@ -120,16 +120,15 @@ module Simp::BeakerHelpers
|
|
120
120
|
FileUtils.remove_entry_secure tmpdir
|
121
121
|
end
|
122
122
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
123
|
+
return unless @results.nil? || @results.empty?
|
124
|
+
File.open(@result_file + '.err', 'w') do |fh|
|
125
|
+
fh.puts(result.stderr.strip)
|
126
|
+
end
|
127
127
|
|
128
|
-
|
129
|
-
|
128
|
+
err_msg = ["Error running inspec command #{inspec_cmd}"]
|
129
|
+
err_msg << "Error captured in #{@result_file}" + '.err'
|
130
130
|
|
131
|
-
|
132
|
-
end
|
131
|
+
raise(err_msg.join("\n"))
|
133
132
|
end
|
134
133
|
|
135
134
|
# Output the report
|
@@ -162,36 +161,34 @@ module Simp::BeakerHelpers
|
|
162
161
|
|
163
162
|
stats = {
|
164
163
|
# Legacy metrics counters for backwards compatibility
|
165
|
-
:
|
166
|
-
:
|
167
|
-
:
|
168
|
-
:
|
164
|
+
failed: 0,
|
165
|
+
passed: 0,
|
166
|
+
skipped: 0,
|
167
|
+
overridden: 0,
|
169
168
|
# End legacy stuff
|
170
|
-
:
|
171
|
-
:
|
172
|
-
:
|
173
|
-
:
|
174
|
-
:
|
169
|
+
global: {
|
170
|
+
failed: [],
|
171
|
+
passed: [],
|
172
|
+
skipped: [],
|
173
|
+
overridden: []
|
175
174
|
},
|
176
|
-
:
|
177
|
-
:
|
178
|
-
:
|
175
|
+
score: 0,
|
176
|
+
report: nil,
|
177
|
+
profiles: {}
|
179
178
|
}
|
180
179
|
|
181
180
|
if results.is_a?(String)
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
fail("Error: Could not read results file at #{results}")
|
186
|
-
end
|
181
|
+
raise("Error: Could not read results file at #{results}") unless File.readable?(results)
|
182
|
+
profiles = JSON.parse(File.read(results))['profiles']
|
183
|
+
|
187
184
|
elsif results.is_a?(Hash)
|
188
185
|
profiles = results['profiles']
|
189
186
|
else
|
190
|
-
|
187
|
+
raise('Error: first argument must be a String path to a file or a Hash')
|
191
188
|
end
|
192
189
|
|
193
190
|
if !profiles || profiles.empty?
|
194
|
-
|
191
|
+
raise("Error: Could not find 'profiles' in the passed results")
|
195
192
|
end
|
196
193
|
|
197
194
|
profiles.each do |profile|
|
@@ -200,7 +197,7 @@ module Simp::BeakerHelpers
|
|
200
197
|
next unless profile_name
|
201
198
|
|
202
199
|
stats[:profiles][profile_name] = {
|
203
|
-
:
|
200
|
+
controls: {}
|
204
201
|
}
|
205
202
|
|
206
203
|
profile['controls'].each do |control|
|
@@ -208,7 +205,7 @@ module Simp::BeakerHelpers
|
|
208
205
|
|
209
206
|
next unless title
|
210
207
|
|
211
|
-
base_title = title.scan(
|
208
|
+
base_title = title.scan(%r{.{1,60}\W|.{1,60}}).map(&:strip).join("\n ")
|
212
209
|
|
213
210
|
if control['results'] && (control['results'].size > 1)
|
214
211
|
control['results'].each do |result|
|
@@ -221,12 +218,12 @@ module Simp::BeakerHelpers
|
|
221
218
|
|
222
219
|
stats[:profiles][profile_name][:controls][full_title][:formatted_title] = formatted_title
|
223
220
|
|
224
|
-
if result['status']
|
221
|
+
if %r{^fail}.match?(result['status'])
|
225
222
|
status = :failed
|
226
|
-
|
223
|
+
'red'
|
227
224
|
else
|
228
225
|
status = :passed
|
229
|
-
|
226
|
+
'green'
|
230
227
|
end
|
231
228
|
|
232
229
|
stats[:global][status] << formatted_title.color
|
@@ -245,8 +242,8 @@ module Simp::BeakerHelpers
|
|
245
242
|
status = :passed
|
246
243
|
color = 'green'
|
247
244
|
|
248
|
-
control['results'].each do |
|
249
|
-
if results['status']
|
245
|
+
control['results'].each do |_result|
|
246
|
+
if %r{^fail}.match?(results['status'])
|
250
247
|
status = :failed
|
251
248
|
color = 'red'
|
252
249
|
end
|
@@ -273,15 +270,15 @@ module Simp::BeakerHelpers
|
|
273
270
|
end
|
274
271
|
|
275
272
|
status_colors = {
|
276
|
-
:
|
277
|
-
:
|
278
|
-
:
|
279
|
-
:
|
273
|
+
failed: 'red',
|
274
|
+
passed: 'green',
|
275
|
+
skipped: 'yellow',
|
276
|
+
overridden: 'white'
|
280
277
|
}
|
281
278
|
|
282
279
|
report = []
|
283
280
|
|
284
|
-
stats[:profiles].
|
281
|
+
stats[:profiles].each_key do |profile|
|
285
282
|
report << "Profile: #{profile}"
|
286
283
|
|
287
284
|
stats[:profiles][profile][:controls].each do |control|
|
@@ -311,14 +308,14 @@ module Simp::BeakerHelpers
|
|
311
308
|
stats[:skipped] = num_skipped
|
312
309
|
stats[:overridden] = num_overridden
|
313
310
|
|
314
|
-
report <<
|
311
|
+
report << 'Statistics:'
|
315
312
|
report << " * Passed: #{num_passed.to_s.green}"
|
316
313
|
report << " * Failed: #{num_failed.to_s.red}"
|
317
314
|
report << " * Skipped: #{num_skipped.to_s.yellow}"
|
318
315
|
|
319
316
|
score = 0
|
320
317
|
if (stats[:global][:passed].count + stats[:global][:failed].count) > 0
|
321
|
-
score = ((stats[:global][:passed].count.to_f/(stats[:global][:passed].count + stats[:global][:failed].count)) * 100.0).round(0)
|
318
|
+
score = ((stats[:global][:passed].count.to_f / (stats[:global][:passed].count + stats[:global][:failed].count)) * 100.0).round(0)
|
322
319
|
end
|
323
320
|
|
324
321
|
report << "\n Score: #{score}%"
|
@@ -326,7 +323,7 @@ module Simp::BeakerHelpers
|
|
326
323
|
stats[:score] = score
|
327
324
|
stats[:report] = report.join("\n")
|
328
325
|
|
329
|
-
|
326
|
+
stats
|
330
327
|
end
|
331
328
|
end
|
332
329
|
end
|
@@ -1,156 +1,148 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
)
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
1
|
+
# Helpers for managing Vagrant snapshots
|
2
|
+
class Simp::BeakerHelpers::Snapshot
|
3
|
+
# The name of the base snapshot that is created if no snapshots currently exist
|
4
|
+
BASE_NAME = '_simp_beaker_base'.freeze
|
5
|
+
|
6
|
+
# Save a snapshot
|
7
|
+
#
|
8
|
+
# @param host [Beaker::Host]
|
9
|
+
# The SUT to work on
|
10
|
+
#
|
11
|
+
# @param snapshot_name [String]
|
12
|
+
# The string to add to the snapshot
|
13
|
+
#
|
14
|
+
def self.save(host, snapshot_name)
|
15
|
+
return unless enabled?
|
16
|
+
vdir = vagrant_dir(host)
|
17
|
+
|
18
|
+
return unless vdir
|
19
|
+
Dir.chdir(vdir) do
|
20
|
+
save(host, BASE_NAME) unless exist?(host, BASE_NAME)
|
21
|
+
|
22
|
+
snap = "#{host.name}_#{snapshot_name}"
|
23
|
+
|
24
|
+
output = `vagrant snapshot save --force #{host.name} "#{snap}"`
|
25
|
+
|
26
|
+
logger.notify(output)
|
27
|
+
|
28
|
+
retry_on(
|
29
|
+
host,
|
30
|
+
%(echo "saving snapshot '#{snap}'" > /dev/null),
|
31
|
+
max_retries: 30,
|
32
|
+
retry_interval: 1,
|
33
|
+
)
|
38
34
|
end
|
35
|
+
end
|
39
36
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
37
|
+
# Whether or not a named snapshot exists
|
38
|
+
#
|
39
|
+
# @param host [Beaker::Host]
|
40
|
+
# The SUT to work on
|
41
|
+
#
|
42
|
+
# @param snapshot_name [String]
|
43
|
+
# The string to add to the snapshot
|
44
|
+
#
|
45
|
+
# @return [Boolean]
|
46
|
+
def self.exist?(host, name)
|
47
|
+
list(host).include?(name)
|
48
|
+
end
|
52
49
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
end
|
50
|
+
# List all snapshots for the given host
|
51
|
+
#
|
52
|
+
# @parma host [Beaker::Host]
|
53
|
+
# The SUT to work on
|
54
|
+
#
|
55
|
+
# @return [Array[String]]
|
56
|
+
# A list of snapshot names for the host
|
57
|
+
def self.list(host)
|
58
|
+
output = []
|
59
|
+
vdir = vagrant_dir(host)
|
60
|
+
|
61
|
+
if vdir
|
62
|
+
Dir.chdir(vdir) do
|
63
|
+
output = `vagrant snapshot list #{host.name}`.lines
|
64
|
+
output.map! do |x|
|
65
|
+
x.split(%r{^#{host.name}_}).last.split(':').first.delete('==>').strip
|
70
66
|
end
|
71
67
|
end
|
72
|
-
|
73
|
-
output
|
74
68
|
end
|
75
69
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
raise output
|
100
|
-
end
|
101
|
-
|
102
|
-
logger.notify(output)
|
103
|
-
|
104
|
-
retry_on(
|
105
|
-
host,
|
106
|
-
%(echo "restoring snapshot '#{snap}'" > /dev/null),
|
107
|
-
:max_retries => 30,
|
108
|
-
:retry_interval => 1
|
109
|
-
)
|
110
|
-
end
|
111
|
-
end
|
70
|
+
output
|
71
|
+
end
|
72
|
+
|
73
|
+
# Restore a snapshot
|
74
|
+
#
|
75
|
+
# @param host [Beaker::Host]
|
76
|
+
# The SUT to work on
|
77
|
+
#
|
78
|
+
# @param snapshot_name [String]
|
79
|
+
# The name that was added to the snapshot
|
80
|
+
#
|
81
|
+
def self.restore(host, snapshot_name)
|
82
|
+
return unless enabled?
|
83
|
+
vdir = vagrant_dir(host)
|
84
|
+
|
85
|
+
return unless vdir
|
86
|
+
Dir.chdir(vdir) do
|
87
|
+
snap = "#{host.name}_#{snapshot_name}"
|
88
|
+
|
89
|
+
output = `vagrant snapshot restore #{host.name} "#{snap}" 2>&1`
|
90
|
+
|
91
|
+
if (output =~ %r{error}i) && output.include?('child')
|
92
|
+
raise output
|
112
93
|
end
|
113
|
-
end
|
114
94
|
|
115
|
-
|
116
|
-
|
117
|
-
# @param host [Beaker::Host]
|
118
|
-
# The SUT to work on
|
119
|
-
#
|
120
|
-
def self.restore_to_base(host)
|
121
|
-
if exist?(host, BASE_NAME)
|
122
|
-
restore(host, BASE_NAME)
|
123
|
-
else
|
124
|
-
save(host, BASE_NAME)
|
95
|
+
if %r{snapshot.*not found}.match?(output)
|
96
|
+
raise output
|
125
97
|
end
|
126
|
-
end
|
127
98
|
|
128
|
-
|
99
|
+
logger.notify(output)
|
129
100
|
|
130
|
-
|
131
|
-
|
101
|
+
retry_on(
|
102
|
+
host,
|
103
|
+
%(echo "restoring snapshot '#{snap}'" > /dev/null),
|
104
|
+
max_retries: 30,
|
105
|
+
retry_interval: 1,
|
106
|
+
)
|
107
|
+
end
|
108
|
+
end
|
132
109
|
|
133
|
-
|
134
|
-
|
135
|
-
|
110
|
+
# Restore all the way back to the base image
|
111
|
+
#
|
112
|
+
# @param host [Beaker::Host]
|
113
|
+
# The SUT to work on
|
114
|
+
#
|
115
|
+
def self.restore_to_base(host)
|
116
|
+
if exist?(host, BASE_NAME)
|
117
|
+
restore(host, BASE_NAME)
|
118
|
+
else
|
119
|
+
save(host, BASE_NAME)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.enabled?
|
124
|
+
enabled = ENV['BEAKER_simp_snapshot'] == 'yes'
|
136
125
|
|
137
|
-
|
126
|
+
unless enabled
|
127
|
+
logger.warn('Snapshotting not enabled, set BEAKER_simp_snapshot=yes to enable')
|
138
128
|
end
|
139
129
|
|
140
|
-
|
141
|
-
|
130
|
+
enabled
|
131
|
+
end
|
142
132
|
|
143
|
-
|
144
|
-
|
133
|
+
def self.vagrant_dir(host)
|
134
|
+
tgt_dir = nil
|
145
135
|
|
146
|
-
|
147
|
-
|
148
|
-
else
|
149
|
-
logger.notify("Could not find local vagrant dir at #{vdir}")
|
150
|
-
end
|
151
|
-
end
|
136
|
+
if host&.options && host.options[:hosts_file]
|
137
|
+
vdir = File.join('.vagrant', 'beaker_vagrant_files', File.basename(host.options[:hosts_file]))
|
152
138
|
|
153
|
-
|
139
|
+
if File.directory?(vdir)
|
140
|
+
tgt_dir = vdir
|
141
|
+
else
|
142
|
+
logger.notify("Could not find local vagrant dir at #{vdir}")
|
143
|
+
end
|
154
144
|
end
|
145
|
+
|
146
|
+
tgt_dir
|
155
147
|
end
|
156
148
|
end
|