onceover 3.22.0 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +17 -0
- data/.github/workflows/release.yaml +25 -17
- data/.github/workflows/tests.yaml +30 -4
- data/.gitignore +2 -1
- data/.rubocop.yml +4 -704
- data/.rubocop_todo.yml +828 -0
- data/CHANGELOG.md +1036 -0
- data/Gemfile +7 -1
- data/LICENSE +202 -0
- data/README.md +81 -35
- data/Rakefile +23 -14
- data/features/auto_vendored.feature +27 -0
- data/features/step_definitions/common.rb +5 -5
- data/features/support/cache_helper.rb +0 -1
- data/features/support/command_helper.rb +0 -2
- data/features/support/controlrepo_helper.rb +0 -2
- data/lib/onceover/beaker/spec_helper.rb +7 -7
- data/lib/onceover/beaker.rb +9 -12
- data/lib/onceover/cli/run.rb +1 -0
- data/lib/onceover/controlrepo.rb +19 -24
- data/lib/onceover/deploy.rb +29 -2
- data/lib/onceover/group.rb +1 -3
- data/lib/onceover/logger.rb +3 -3
- data/lib/onceover/node.rb +0 -2
- data/lib/onceover/rake_tasks.rb +9 -4
- data/lib/onceover/rspec/formatters.rb +3 -4
- data/lib/onceover/test.rb +1 -2
- data/lib/onceover/testconfig.rb +2 -2
- data/lib/onceover/vendored_modules.rb +186 -0
- data/onceover.gemspec +24 -21
- data/spec/fixtures/controlrepos/caching/spec/factsets/README.md +1 -1
- data/spec/fixtures/controlrepos/caching/spec/pre_conditions/README.md +1 -1
- data/spec/fixtures/controlrepos/factsets/spec/factsets/README.md +1 -1
- data/spec/fixtures/controlrepos/puppet_controlrepo/Gemfile +1 -1
- data/spec/fixtures/controlrepos/vendored/Puppetfile +3 -0
- data/spec/fixtures/controlrepos/vendored/Puppetfile.cron +5 -0
- data/spec/fixtures/controlrepos/vendored/environment.conf +1 -0
- data/spec/fixtures/controlrepos/vendored/site-modules/role/manifests/cron.pp +9 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/augeas_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/augeas_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/cron_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/cron_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/host_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/host_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/mount_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/mount_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/repo_tree-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/repo_tree-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/scheduled_task-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/scheduled_task-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/selinux_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/selinux_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/sshkeys_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/sshkeys_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/yumrepo_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/yumrepo_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zfs_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zfs_core-puppet_agent-8.6.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zone_core-puppet_agent-7.30.0.json +1 -0
- data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zone_core-puppet_agent-8.6.0.json +1 -0
- data/spec/onceover/controlrepo_spec.rb +1 -1
- data/templates/factsets_README.md.erb +1 -1
- data/templates/pre_conditions_README.md.erb +1 -1
- metadata +56 -9
data/lib/onceover/beaker.rb
CHANGED
@@ -11,9 +11,8 @@ class Onceover
|
|
11
11
|
# se or an array
|
12
12
|
|
13
13
|
if facts.is_a?(Array)
|
14
|
-
returnval =
|
15
|
-
|
16
|
-
returnval << self.facts_to_vagrant_box(fact)
|
14
|
+
returnval = facts.map do |fact|
|
15
|
+
self.facts_to_vagrant_box(fact)
|
17
16
|
end
|
18
17
|
return returnval
|
19
18
|
end
|
@@ -64,9 +63,8 @@ class Onceover
|
|
64
63
|
warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
|
65
64
|
|
66
65
|
if facts.is_a?(Array)
|
67
|
-
returnval =
|
68
|
-
|
69
|
-
returnval << self.facts_to_platform(fact)
|
66
|
+
returnval = facts.map do |fact|
|
67
|
+
self.facts_to_platform(fact)
|
70
68
|
end
|
71
69
|
return returnval
|
72
70
|
end
|
@@ -131,9 +129,9 @@ class Onceover
|
|
131
129
|
|
132
130
|
# Do an r10k deploy
|
133
131
|
r10k_deploy(host, {
|
134
|
-
|
135
|
-
|
136
|
-
|
132
|
+
:puppetfile => true,
|
133
|
+
:configfile => '/tmp/r10k.yaml',
|
134
|
+
})
|
137
135
|
end
|
138
136
|
|
139
137
|
# This actually provisions a node and checks that puppet will be able to run and
|
@@ -153,7 +151,6 @@ class Onceover
|
|
153
151
|
opts = {:check_idempotency => true}.merge(opts)
|
154
152
|
opts = {:deploy_controlrepo => true}.merge(opts)
|
155
153
|
|
156
|
-
|
157
154
|
raise "Hosts must be a single host object, not an array" if host.is_a?(Array)
|
158
155
|
raise "Class must be a single Class [String], not an array" unless puppet_class.is_a?(String)
|
159
156
|
|
@@ -216,8 +213,8 @@ class Onceover
|
|
216
213
|
# to refer to each key as either a string or an object
|
217
214
|
current_opts.default_proc = proc do |h, k|
|
218
215
|
case k
|
219
|
-
|
220
|
-
|
216
|
+
when String then sym = k.to_sym; h[sym] if h.key?(sym)
|
217
|
+
when Symbol then str = k.to_s; h[str] if h.key?(str)
|
221
218
|
end
|
222
219
|
end
|
223
220
|
|
data/lib/onceover/cli/run.rb
CHANGED
@@ -44,6 +44,7 @@ This includes deploying using r10k and running all custom tests.
|
|
44
44
|
optional nil, :format, 'Which RSpec formatter to use, valid options are: documentation, progress, FailureCollector, OnceoverFormatter. You also specify this multiple times', multiple: true, default: :defaults
|
45
45
|
optional nil, :no_workarounds, 'Disables workarounds that have been added for convenience to get around common RSPec issues such as https://github.com/rodjek/rspec-puppet/issues/665'
|
46
46
|
optional :ff, :fail_fast, 'Abort the run after the first failure'
|
47
|
+
optional nil, :auto_vendored, 'Attempt to resolve vendored puppet modules. Ex: puppetlabs/cron_core', default: false
|
47
48
|
|
48
49
|
run do |opts, args, cmd|
|
49
50
|
repo = Onceover::Controlrepo.new(opts)
|
data/lib/onceover/controlrepo.rb
CHANGED
@@ -119,7 +119,7 @@ class Onceover
|
|
119
119
|
@spec_dir = opts[:spec_dir] || File.expand_path('./spec', @root)
|
120
120
|
@facts_dir = opts[:facts_dir] || File.expand_path('factsets', @spec_dir)
|
121
121
|
_facts_dirs = [@facts_dir, File.expand_path('../../factsets', __dir__)]
|
122
|
-
_facts_files = opts[:facts_files]
|
122
|
+
_facts_files = opts[:facts_files] || _facts_dirs.map{|d| File.join(d, '*.json')}
|
123
123
|
@facts_files = _facts_files.map{|_path| Dir[_path]}.flatten
|
124
124
|
|
125
125
|
@nodeset_file = opts[:nodeset_file] || File.expand_path('./spec/acceptance/nodesets/onceover-nodes.yml', @root)
|
@@ -139,7 +139,6 @@ class Onceover
|
|
139
139
|
end
|
140
140
|
end
|
141
141
|
|
142
|
-
|
143
142
|
def to_s
|
144
143
|
require 'colored'
|
145
144
|
|
@@ -182,9 +181,8 @@ class Onceover
|
|
182
181
|
end
|
183
182
|
|
184
183
|
# Get all the classes from all of the manifests
|
185
|
-
classes =
|
186
|
-
|
187
|
-
classes << get_classes(dir)
|
184
|
+
classes = code_dirs.map do |dir|
|
185
|
+
get_classes(dir)
|
188
186
|
end
|
189
187
|
classes.flatten
|
190
188
|
end
|
@@ -208,9 +206,8 @@ class Onceover
|
|
208
206
|
raise "Filter param must be a hash" unless filter.is_a?(Hash)
|
209
207
|
|
210
208
|
all_facts.keep_if do |hash|
|
211
|
-
matches =
|
212
|
-
|
213
|
-
matches << keypair_is_in_hash(hash,filter_fact,value)
|
209
|
+
matches = filter.map do |filter_fact,value|
|
210
|
+
keypair_is_in_hash(hash,filter_fact,value)
|
214
211
|
end
|
215
212
|
!matches.include? false
|
216
213
|
end
|
@@ -231,10 +228,9 @@ class Onceover
|
|
231
228
|
puppetfile.load!
|
232
229
|
|
233
230
|
output_array = []
|
234
|
-
threads = []
|
235
231
|
error_array = []
|
236
|
-
puppetfile.modules.
|
237
|
-
|
232
|
+
threads = puppetfile.modules.map do |mod|
|
233
|
+
Thread.new do
|
238
234
|
begin
|
239
235
|
row = []
|
240
236
|
logger.debug "Loading data for #{mod.full_name}"
|
@@ -268,7 +264,7 @@ class Onceover
|
|
268
264
|
"PatchLevel_minor".green
|
269
265
|
else
|
270
266
|
"No".green
|
271
|
-
|
267
|
+
end
|
272
268
|
|
273
269
|
row << mod.v3_module.endorsement
|
274
270
|
superseded_by = mod.v3_module.superseded_by
|
@@ -290,8 +286,8 @@ class Onceover
|
|
290
286
|
error_array << error
|
291
287
|
logger.debug "Error loading module #{mod.full_name} - #{e.inspect}"
|
292
288
|
end
|
293
|
-
end
|
294
289
|
end
|
290
|
+
end
|
295
291
|
|
296
292
|
threads.map(&:join)
|
297
293
|
|
@@ -313,13 +309,12 @@ class Onceover
|
|
313
309
|
# TODO: Make sure we can deal with :latest
|
314
310
|
|
315
311
|
# Create threading resources
|
316
|
-
|
317
|
-
queue = Queue.new
|
312
|
+
queue = Queue.new
|
318
313
|
queue.push(puppetfile_string)
|
319
314
|
|
320
315
|
puppetfile.modules.keep_if {|m| m.is_a?(R10K::Module::Forge)}
|
321
|
-
puppetfile.modules.
|
322
|
-
|
316
|
+
threads = puppetfile.modules.map do |mod|
|
317
|
+
Thread.new do
|
323
318
|
logger.debug "Getting latest version of #{mod.full_name}"
|
324
319
|
latest_version = mod.v3_module.current_release.version
|
325
320
|
|
@@ -360,14 +355,14 @@ class Onceover
|
|
360
355
|
# Set it up as a symlink, because we are using local files in the Puppetfile
|
361
356
|
symlinks << {
|
362
357
|
'name' => mod.name,
|
363
|
-
'dir'
|
358
|
+
'dir' => mod.expected_version[:path]
|
364
359
|
}
|
365
360
|
elsif mod.expected_version.is_a?(String)
|
366
361
|
# Set it up as a normal forge module
|
367
362
|
forge_modules << {
|
368
363
|
'name' => mod.name,
|
369
364
|
'repo' => mod.title,
|
370
|
-
'ref'
|
365
|
+
'ref' => mod.expected_version
|
371
366
|
}
|
372
367
|
end
|
373
368
|
elsif mod.is_a? R10K::Module::Git
|
@@ -377,7 +372,7 @@ class Onceover
|
|
377
372
|
# I know I shouldn't be doing this, but trust me, there are no methods
|
378
373
|
# anywhere that expose this value, I looked.
|
379
374
|
'repo' => mod.instance_variable_get(:@remote),
|
380
|
-
'ref'
|
375
|
+
'ref' => mod.version
|
381
376
|
}
|
382
377
|
end
|
383
378
|
end
|
@@ -392,7 +387,7 @@ class Onceover
|
|
392
387
|
Dir["#{dir}/*"].each do |mod|
|
393
388
|
symlinks << {
|
394
389
|
'name' => File.basename(mod),
|
395
|
-
'dir'
|
390
|
+
'dir' => Pathname.new(File.expand_path(mod)).relative_path_from(Pathname.new(@root))#File.expand_path(mod)
|
396
391
|
}
|
397
392
|
end
|
398
393
|
end
|
@@ -574,9 +569,9 @@ class Onceover
|
|
574
569
|
# Add the resulting info to the hosts hash. This is what the
|
575
570
|
# template will output
|
576
571
|
hosts_hash[node_name] = {
|
577
|
-
:platform
|
578
|
-
:boxname
|
579
|
-
:url
|
572
|
+
:platform => platform,
|
573
|
+
:boxname => boxname,
|
574
|
+
:url => url,
|
580
575
|
:comment_out => comment_out
|
581
576
|
}
|
582
577
|
end
|
data/lib/onceover/deploy.rb
CHANGED
@@ -13,6 +13,10 @@ class Onceover
|
|
13
13
|
skip_r10k_default = !(File.file?(repo.puppetfile))
|
14
14
|
skip_r10k = opts[:skip_r10k] || skip_r10k_default
|
15
15
|
force = opts[:force] || false
|
16
|
+
# Only attempt to resolve vendored modules if configured to do so
|
17
|
+
auto_vendored = opts[:auto_vendored] || false
|
18
|
+
|
19
|
+
require 'onceover/vendored_modules' if auto_vendored
|
16
20
|
|
17
21
|
if repo.tempdir == nil
|
18
22
|
repo.tempdir = Dir.mktmpdir('r10k')
|
@@ -63,7 +67,11 @@ class Onceover
|
|
63
67
|
# add to list of files copied to cache by onceover
|
64
68
|
onceover_manifest << relative_source
|
65
69
|
|
66
|
-
if File.
|
70
|
+
if File.symlink?(source)
|
71
|
+
# Handle symlinks
|
72
|
+
link_target = File.readlink(source) # Get the target of the symlink
|
73
|
+
FileUtils.ln_s link_target, target, force: true # Create symlink at target
|
74
|
+
elsif File.directory? source
|
67
75
|
Find.prune if excluded_dirs.include? source
|
68
76
|
FileUtils.mkdir target
|
69
77
|
else
|
@@ -90,7 +98,26 @@ class Onceover
|
|
90
98
|
if /:control_branch/.match(puppetfile_contents)
|
91
99
|
logger.debug "replacing :control_branch mentions in the Puppetfile with #{git_branch}"
|
92
100
|
new_puppetfile_contents = puppetfile_contents.gsub(":control_branch", "'#{git_branch}'")
|
93
|
-
File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
|
101
|
+
File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
|
102
|
+
end
|
103
|
+
|
104
|
+
if auto_vendored
|
105
|
+
tmp_puppetfile = File.join(temp_controlrepo, 'Puppetfile')
|
106
|
+
tmp_puppetfile_contents = File.read(tmp_puppetfile)
|
107
|
+
vm = Onceover::VendoredModules.new({ repo: repo })
|
108
|
+
puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: temp_controlrepo)
|
109
|
+
vm.puppetfile_missing_vendored(puppetfile)
|
110
|
+
unless vm.missing_vendored.empty?
|
111
|
+
missing_slugs = vm.missing_vendored.map do |missing_mod|
|
112
|
+
missing_mod.keys[0]
|
113
|
+
end
|
114
|
+
logger.debug "Adding #{missing_slugs} to #{tmp_puppetfile}"
|
115
|
+
modlines = vm.missing_vendored.map do |missing_mod|
|
116
|
+
mod_slug = missing_mod.keys[0]
|
117
|
+
"mod '#{mod_slug}',\n git: '#{missing_mod[mod_slug][:git]}',\n ref: '#{missing_mod[mod_slug][:ref]}'"
|
118
|
+
end.join("\n")
|
119
|
+
File.write(tmp_puppetfile, "#{tmp_puppetfile_contents}\n# Onceover Managed Vendored Modules\n#{modlines}")
|
120
|
+
end
|
94
121
|
end
|
95
122
|
end
|
96
123
|
|
data/lib/onceover/group.rb
CHANGED
@@ -28,10 +28,8 @@ class Onceover
|
|
28
28
|
@members = []
|
29
29
|
else
|
30
30
|
# Turn it into a full list
|
31
|
-
member_objects = []
|
32
|
-
|
33
31
|
# This should also handle lists that include groups
|
34
|
-
members.
|
32
|
+
member_objects = members.map { |member| Onceover::TestConfig.find_list(member) }
|
35
33
|
member_objects.flatten!
|
36
34
|
|
37
35
|
# Check that they are all the same type
|
data/lib/onceover/logger.rb
CHANGED
@@ -8,8 +8,8 @@ module Onceover::Logger
|
|
8
8
|
'bright',
|
9
9
|
:levels => {
|
10
10
|
:debug => :cyan,
|
11
|
-
:info
|
12
|
-
:warn
|
11
|
+
:info => :green,
|
12
|
+
:warn => :yellow,
|
13
13
|
:error => :red,
|
14
14
|
:fatal => [:white, :on_red]
|
15
15
|
}
|
@@ -18,7 +18,7 @@ module Onceover::Logger
|
|
18
18
|
Logging.appenders.stdout(
|
19
19
|
'stdout',
|
20
20
|
:layout => Logging.layouts.pattern(
|
21
|
-
:pattern
|
21
|
+
:pattern => '%l\t -> %m\n',
|
22
22
|
:color_scheme => 'bright'
|
23
23
|
)
|
24
24
|
)
|
data/lib/onceover/node.rb
CHANGED
data/lib/onceover/rake_tasks.rb
CHANGED
@@ -4,7 +4,6 @@ require 'pathname'
|
|
4
4
|
@repo = nil
|
5
5
|
@config = nil
|
6
6
|
|
7
|
-
|
8
7
|
desc 'Writes a `fixtures.yml` file based on the Puppetfile'
|
9
8
|
task :generate_fixtures do
|
10
9
|
repo = Onceover::Controlrepo.new
|
@@ -15,7 +14,6 @@ task :generate_fixtures do
|
|
15
14
|
File.write(File.expand_path('./.fixtures.yml', repo.root), repo.fixtures)
|
16
15
|
end
|
17
16
|
|
18
|
-
|
19
17
|
desc "Modifies your `hiera.yaml` to point at the hieradata relative to its position."
|
20
18
|
task :hiera_setup do
|
21
19
|
repo = Onceover::Controlrepo.new
|
@@ -45,7 +43,6 @@ task :generate_onceover_yaml do
|
|
45
43
|
puts ERB.new(onceover_yaml_template, nil, '-').result(binding)
|
46
44
|
end
|
47
45
|
|
48
|
-
|
49
46
|
task :generate_nodesets do
|
50
47
|
warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
|
51
48
|
|
@@ -75,7 +72,7 @@ task :generate_nodesets do
|
|
75
72
|
comment_out = false
|
76
73
|
box_info = MultiJson.load(response.body)
|
77
74
|
box_info['current_version']['providers'].each do |provider|
|
78
|
-
if
|
75
|
+
if provider['name'] == 'virtualbox'
|
79
76
|
url = provider['original_url']
|
80
77
|
end
|
81
78
|
end
|
@@ -86,5 +83,13 @@ task :generate_nodesets do
|
|
86
83
|
fixtures_template = File.read(File.expand_path('./nodeset.yaml.erb', template_dir))
|
87
84
|
puts ERB.new(fixtures_template, nil, '-').result(binding)
|
88
85
|
end
|
86
|
+
end
|
87
|
+
|
88
|
+
desc 'Create cache for vendored modules'
|
89
|
+
task :generate_vendor_cache do
|
90
|
+
require 'onceover/controlrepo'
|
91
|
+
require 'onceover/vendored_modules'
|
89
92
|
|
93
|
+
repo = Onceover::Controlrepo.new(debug: true)
|
94
|
+
Onceover::VendoredModules.new({ repo: repo, cachedir: File.join(repo.spec_dir, 'vendored_modules'), force_update: true })
|
90
95
|
end
|
@@ -127,9 +127,9 @@ class OnceoverFormatter
|
|
127
127
|
|
128
128
|
scanned_errors.map do |error_matches|
|
129
129
|
{
|
130
|
-
text:
|
131
|
-
file:
|
132
|
-
line:
|
130
|
+
text: error_matches[0],
|
131
|
+
file: calculate_relative_source(error_matches[1]),
|
132
|
+
line: error_matches[2],
|
133
133
|
column: error_matches[3],
|
134
134
|
}
|
135
135
|
end
|
@@ -223,7 +223,6 @@ class OnceoverFormatter
|
|
223
223
|
def longest_group
|
224
224
|
RSpec.configuration.world.example_groups.max { |a,b| a.description.length <=> b.description.length}.description.length
|
225
225
|
end
|
226
|
-
|
227
226
|
end
|
228
227
|
|
229
228
|
class OnceoverFormatterParallel < OnceoverFormatter
|
data/lib/onceover/test.rb
CHANGED
@@ -12,9 +12,8 @@ class Onceover
|
|
12
12
|
# it will then detect them and expand them out into their respective objects so that
|
13
13
|
# we just end up with a list of nodes and classes
|
14
14
|
def initialize(on_this, test_this, test_config)
|
15
|
-
|
16
15
|
@default_test_config = {
|
17
|
-
'check_idempotency'
|
16
|
+
'check_idempotency' => true,
|
18
17
|
'runs_before_idempotency' => 1
|
19
18
|
}
|
20
19
|
|
data/lib/onceover/testconfig.rb
CHANGED
@@ -282,9 +282,9 @@ class Onceover
|
|
282
282
|
def run_filters(tests)
|
283
283
|
# All of this needs to be applied AFTER deduplication but BEFORE writing
|
284
284
|
filters = {
|
285
|
-
'tags'
|
285
|
+
'tags' => @filter_tags,
|
286
286
|
'classes' => @filter_classes,
|
287
|
-
'nodes'
|
287
|
+
'nodes' => @filter_nodes
|
288
288
|
}
|
289
289
|
filters.each do |method, filter_list|
|
290
290
|
if filter_list
|
@@ -0,0 +1,186 @@
|
|
1
|
+
require 'puppet/version'
|
2
|
+
require 'net/http'
|
3
|
+
require 'uri'
|
4
|
+
require 'multi_json'
|
5
|
+
require 'r10k/module_loader/puppetfile'
|
6
|
+
require 'onceover/logger'
|
7
|
+
|
8
|
+
### operations
|
9
|
+
#
|
10
|
+
# 1. resolve all the component json files in the puppet-agent repo for vendored modules
|
11
|
+
# 2. parse each json file and determine vendored modules repo + ref
|
12
|
+
#
|
13
|
+
###
|
14
|
+
|
15
|
+
## Example
|
16
|
+
#
|
17
|
+
# vm = Onceover::VendoredModules.new
|
18
|
+
# puts vm.vendored_references
|
19
|
+
# puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: '.')
|
20
|
+
# vm.puppetfile_missing_vendored(puppetfile)
|
21
|
+
# puts vm.missing_vendored.inspect
|
22
|
+
|
23
|
+
class Onceover
|
24
|
+
class VendoredModules
|
25
|
+
attr_reader :vendored_references, :missing_vendored
|
26
|
+
|
27
|
+
def initialize(opts = {})
|
28
|
+
@repo = opts[:repo] || Onceover::Controlrepo.new
|
29
|
+
@cachedir = opts[:cachedir] || File.join(@repo.tempdir, 'vendored_modules')
|
30
|
+
@puppet_version = Gem::Version.new(Puppet.version)
|
31
|
+
@puppet_major_version = Gem::Version.new(@puppet_version.segments[0])
|
32
|
+
@force_update = opts[:force_update] || false
|
33
|
+
|
34
|
+
@missing_vendored = []
|
35
|
+
|
36
|
+
# This only applies to puppet >= 6 so bail early
|
37
|
+
raise 'Auto resolving vendored modules only applies to puppet versions >= 6' unless @puppet_major_version >= Gem::Version.new('6')
|
38
|
+
|
39
|
+
# Create cachedir
|
40
|
+
unless File.directory?(@cachedir)
|
41
|
+
logger.debug "Creating #{@cachedir}"
|
42
|
+
FileUtils.mkdir_p(@cachedir)
|
43
|
+
end
|
44
|
+
|
45
|
+
# Location of user provided caches:
|
46
|
+
# control-repo/spec/vendored_modules/<component>-puppet_agent-<agent version>.json
|
47
|
+
@manual_vendored_dir = File.join(@repo.spec_dir, 'vendored_modules')
|
48
|
+
|
49
|
+
# Get the entire file tree of the puppetlabs/puppet-agent repository
|
50
|
+
# https://docs.github.com/en/rest/git/trees?apiVersion=2022-11-28#get-a-tree
|
51
|
+
puppet_agent_tree = query_or_cache(
|
52
|
+
"https://api.github.com/repos/puppetlabs/puppet-agent/git/trees/#{@puppet_version}",
|
53
|
+
{ recursive: true },
|
54
|
+
component_cache('repo_tree'),
|
55
|
+
)
|
56
|
+
# Get only the module-puppetlabs-<something>_core.json component files
|
57
|
+
vendored_components = puppet_agent_tree['tree'].select { |file| %r{configs/components/module-puppetlabs-\w+\.json}.match(file['path']) }
|
58
|
+
# Get the contents of each component file
|
59
|
+
# https://docs.github.com/en/rest/git/blobs?apiVersion=2022-11-28#get-a-blob
|
60
|
+
@vendored_references = vendored_components.map do |component|
|
61
|
+
mod_slug = component['path'].match(/.*(puppetlabs-\w+).json$/)[1]
|
62
|
+
mod_name = mod_slug.match(/puppetlabs-(\w+)/)[1]
|
63
|
+
query_or_cache(
|
64
|
+
component['url'],
|
65
|
+
nil,
|
66
|
+
component_cache(mod_name),
|
67
|
+
)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def component_cache(component)
|
72
|
+
# Ideally we want a cache for the version of the puppet agent used in tests
|
73
|
+
desired_name = "#{component}-puppet_agent-#{@puppet_version}.json"
|
74
|
+
# By default look for any caches created during previous runs
|
75
|
+
cache_file = File.join(@cachedir, desired_name)
|
76
|
+
|
77
|
+
# If the user provides their own cache
|
78
|
+
if !@force_update && File.directory?(@manual_vendored_dir)
|
79
|
+
# Check for any '<component>-puppet_agent-<puppet version>.json' files
|
80
|
+
dg = Dir.glob(File.join(@manual_vendored_dir, "#{component}-puppet_agent*"))
|
81
|
+
# Check if there are multiple versions of the component cache
|
82
|
+
if dg.size > 1
|
83
|
+
# If there is the same version supplied as whats being tested against use that
|
84
|
+
if dg.any? { |s| s[desired_name] }
|
85
|
+
cache_file = File.join(@manual_vendored_dir, desired_name)
|
86
|
+
# If there are any with the same major version, use the latest supplied
|
87
|
+
elsif dg.any? { |s| s["#{component}-puppet_agent-#{@puppet_major_version}"] }
|
88
|
+
maj_match = dg.select { |f| /#{component}-puppet_agent-#{@puppet_major_version}.\d+\.\d+\.json/.match(f) }
|
89
|
+
maj_match.each do |f|
|
90
|
+
next unless (version_from_file(cache_file) == version_from_file(desired_name)) || (version_from_file(f) >= version_from_file(cache_file))
|
91
|
+
|
92
|
+
# if the current cache version matches the desired version, use the first matching major version in user cache
|
93
|
+
# if there are multiple major version matches in user cache, use the latest
|
94
|
+
cache_file = f
|
95
|
+
end
|
96
|
+
# Otherwise just use the latest supplied
|
97
|
+
else
|
98
|
+
dg.each { |f| cache_file = f if version_from_file(f) >= version_from_file(cache_file) }
|
99
|
+
end
|
100
|
+
# If there is only one use that
|
101
|
+
elsif dg.size == 1
|
102
|
+
cache_file = dg[0]
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# Warn the user if cached version does not match whats being used to test
|
107
|
+
cache_version = version_from_file(cache_file)
|
108
|
+
logger.warn "Cache for #{component} is for puppet_agent #{cache_version}, while you are testing against puppet_agent #{@puppet_version}. Consider updating your cache to ensure consistent behavior in your tests" if cache_version != @puppet_version
|
109
|
+
|
110
|
+
cache_file
|
111
|
+
end
|
112
|
+
|
113
|
+
def version_from_file(cache_file)
|
114
|
+
version_regex = /.*-puppet_agent-(\d+\.\d+\.\d+)\.json/
|
115
|
+
Gem::Version.new(version_regex.match(cache_file)[1])
|
116
|
+
end
|
117
|
+
|
118
|
+
# Currently expects to be passed a R10K::Puppetfile object.
|
119
|
+
# ex: R10K::ModuleLoader::Puppetfile.new(basedir: '.')
|
120
|
+
def puppetfile_missing_vendored(puppetfile)
|
121
|
+
puppetfile.load
|
122
|
+
@vendored_references.each do |mod|
|
123
|
+
# Extract name and slug from url
|
124
|
+
mod_slug = mod['url'].match(/.*(puppetlabs-\w+)\.git/)[1]
|
125
|
+
mod_name = mod_slug.match(/^puppetlabs-(\w+)$/)[1]
|
126
|
+
# Array of modules whos names match
|
127
|
+
existing = puppetfile.modules.select { |e_mod| e_mod.name == mod_name }
|
128
|
+
if existing.empty?
|
129
|
+
# Change url to https instead of ssh to allow anonymous git clones
|
130
|
+
# so that users do not need to have an ssh keypair associated with a Github account
|
131
|
+
url = mod['url'].gsub('git@github.com:', 'https://github.com/')
|
132
|
+
@missing_vendored << { mod_slug => { git: url, ref: mod['ref'] } }
|
133
|
+
logger.debug "#{mod_name} found to be missing in Puppetfile"
|
134
|
+
else
|
135
|
+
logger.debug "#{mod_name} found in Puppetfile. Using the specified version"
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# Return json from a query whom caches, or from the cache to avoid spamming github
|
141
|
+
def query_or_cache(url, params, filepath)
|
142
|
+
if (File.exist? filepath) && (@force_update == false)
|
143
|
+
logger.debug "Using cache: #{filepath}"
|
144
|
+
json = read_json_dump(filepath)
|
145
|
+
else
|
146
|
+
logger.debug "Making GET request to: #{url}"
|
147
|
+
json = github_get(url, params)
|
148
|
+
logger.debug "Caching response to: #{filepath}"
|
149
|
+
write_json_dump(filepath, json)
|
150
|
+
end
|
151
|
+
json
|
152
|
+
end
|
153
|
+
|
154
|
+
# Given a github url and optional query parameters, return the parsed json body
|
155
|
+
def github_get(url, params)
|
156
|
+
uri = URI.parse(url)
|
157
|
+
uri.query = URI.encode_www_form(params) if params
|
158
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
159
|
+
http.use_ssl = (uri.scheme == 'https')
|
160
|
+
request = Net::HTTP::Get.new(uri.request_uri)
|
161
|
+
request['Accept'] = 'application/vnd.github.raw+json'
|
162
|
+
request['X-GitHub-Api-Version'] = '2022-11-28'
|
163
|
+
response = http.request(request)
|
164
|
+
|
165
|
+
case response
|
166
|
+
when Net::HTTPOK # 200
|
167
|
+
MultiJson.load(response.body)
|
168
|
+
else
|
169
|
+
# Expose the ratelimit response headers
|
170
|
+
# https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
|
171
|
+
ratelimit_headers = response.to_hash.select { |k, _v| k =~ /x-ratelimit.*/ }
|
172
|
+
raise "#{response.code} #{response.message} #{ratelimit_headers}"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
# Returns parsed json of file
|
177
|
+
def read_json_dump(filepath)
|
178
|
+
MultiJson.load(File.read(filepath))
|
179
|
+
end
|
180
|
+
|
181
|
+
# Writes json to a file
|
182
|
+
def write_json_dump(filepath, json_data)
|
183
|
+
File.write(filepath, MultiJson.dump(json_data))
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
data/onceover.gemspec
CHANGED
@@ -4,10 +4,10 @@ $LOAD_PATH.unshift File.expand_path('lib', __dir__)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |s| # rubocop:disable Gemspec/RequireMFA
|
6
6
|
s.name = "onceover"
|
7
|
-
s.version = "
|
8
|
-
s.authors = ["Dylan Ratcliffe"]
|
9
|
-
s.email = ["
|
10
|
-
s.homepage = "https://github.com/
|
7
|
+
s.version = "4.0.0"
|
8
|
+
s.authors = ["Dylan Ratcliffe", 'Vox Pupuli']
|
9
|
+
s.email = ["voxpupuli@groups.io"]
|
10
|
+
s.homepage = "https://github.com/voxpupuli/onceover"
|
11
11
|
s.summary = "Testing tools for Puppet controlrepos"
|
12
12
|
s.description = "Automatically generates tests for your Puppet code"
|
13
13
|
s.licenses = 'Apache-2.0'
|
@@ -17,22 +17,25 @@ Gem::Specification.new do |s| # rubocop:disable Gemspec/RequireMFA
|
|
17
17
|
s.bindir = 'bin'
|
18
18
|
s.executables = 'onceover'
|
19
19
|
|
20
|
+
s.required_ruby_version = Gem::Requirement.new('>= 2.7')
|
21
|
+
|
20
22
|
# Runtime dependencies, but also probably dependencies of requiring projects
|
21
|
-
s.
|
22
|
-
s.
|
23
|
-
s.
|
24
|
-
s.
|
25
|
-
s.
|
26
|
-
s.
|
27
|
-
s.
|
28
|
-
s.
|
29
|
-
s.
|
30
|
-
s.
|
31
|
-
s.
|
32
|
-
s.
|
33
|
-
s.
|
34
|
-
s.
|
35
|
-
s.
|
36
|
-
s.
|
37
|
-
s.
|
23
|
+
s.add_dependency 'backticks', '>= 1.0.2'
|
24
|
+
s.add_dependency 'colored', '>= 1.2'
|
25
|
+
s.add_dependency 'cri', '>= 2.6'
|
26
|
+
s.add_dependency 'deep_merge', '>= 1.0.0'
|
27
|
+
s.add_dependency 'git'
|
28
|
+
s.add_dependency 'logging', '>= 2.0.0'
|
29
|
+
s.add_dependency 'multi_json', '>= 1.10'
|
30
|
+
s.add_dependency 'parallel_tests', ">= 2.0.0"
|
31
|
+
s.add_dependency 'puppet', '>=4.0'
|
32
|
+
s.add_dependency 'puppetlabs_spec_helper', ">= 0.4.0"
|
33
|
+
s.add_dependency 'r10k', '>=2.1.0'
|
34
|
+
s.add_dependency 'rake', '>= 10.0.0'
|
35
|
+
s.add_dependency 'rspec', '>= 3.0.0'
|
36
|
+
s.add_dependency 'rspec_junit_formatter', '>= 0.2.0'
|
37
|
+
s.add_dependency 'rspec-puppet', ">= 2.4.0"
|
38
|
+
s.add_dependency 'terminal-table', '>= 1.8.0'
|
39
|
+
s.add_dependency 'versionomy', '>= 0.5.0'
|
40
|
+
s.add_development_dependency 'voxpupuli-rubocop', '~> 3.0.0'
|
38
41
|
end
|
@@ -4,4 +4,4 @@ This directory is where we put any custom factsets that we want to use. They can
|
|
4
4
|
|
5
5
|
**Hot tip:** If you already have factsets in here when you run `onceover init` they will be picked up and added to the config file Automatically
|
6
6
|
|
7
|
-
More info: https://github.com/
|
7
|
+
More info: https://github.com/voxpupuli/onceover#factsets
|
@@ -21,4 +21,4 @@ service { 'pe-puppetserver':
|
|
21
21
|
|
22
22
|
This will mean that the `pe-puppetserver` service is in the catalog for spec testing and will even allow you to try to restart it during acceptance tests without the service actually being present.
|
23
23
|
|
24
|
-
More info: https://github.com/
|
24
|
+
More info: https://github.com/voxpupuli/onceover#using-workarounds
|
@@ -4,4 +4,4 @@ This directory is where we put any custom factsets that we want to use. They can
|
|
4
4
|
|
5
5
|
**Hot tip:** If you already have factsets in here when you run `onceover init` they will be picked up and added to the config file Automatically
|
6
6
|
|
7
|
-
More info: https://github.com/
|
7
|
+
More info: https://github.com/voxpupuli/onceover#factsets
|