onceover 3.22.0 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +17 -0
  3. data/.github/workflows/release.yaml +25 -17
  4. data/.github/workflows/tests.yaml +30 -4
  5. data/.gitignore +2 -1
  6. data/.rubocop.yml +4 -704
  7. data/.rubocop_todo.yml +828 -0
  8. data/CHANGELOG.md +1036 -0
  9. data/Gemfile +7 -1
  10. data/LICENSE +202 -0
  11. data/README.md +81 -35
  12. data/Rakefile +23 -14
  13. data/features/auto_vendored.feature +27 -0
  14. data/features/step_definitions/common.rb +5 -5
  15. data/features/support/cache_helper.rb +0 -1
  16. data/features/support/command_helper.rb +0 -2
  17. data/features/support/controlrepo_helper.rb +0 -2
  18. data/lib/onceover/beaker/spec_helper.rb +7 -7
  19. data/lib/onceover/beaker.rb +9 -12
  20. data/lib/onceover/cli/run.rb +1 -0
  21. data/lib/onceover/controlrepo.rb +19 -24
  22. data/lib/onceover/deploy.rb +29 -2
  23. data/lib/onceover/group.rb +1 -3
  24. data/lib/onceover/logger.rb +3 -3
  25. data/lib/onceover/node.rb +0 -2
  26. data/lib/onceover/rake_tasks.rb +9 -4
  27. data/lib/onceover/rspec/formatters.rb +3 -4
  28. data/lib/onceover/test.rb +1 -2
  29. data/lib/onceover/testconfig.rb +2 -2
  30. data/lib/onceover/vendored_modules.rb +186 -0
  31. data/onceover.gemspec +24 -21
  32. data/spec/fixtures/controlrepos/caching/spec/factsets/README.md +1 -1
  33. data/spec/fixtures/controlrepos/caching/spec/pre_conditions/README.md +1 -1
  34. data/spec/fixtures/controlrepos/factsets/spec/factsets/README.md +1 -1
  35. data/spec/fixtures/controlrepos/puppet_controlrepo/Gemfile +1 -1
  36. data/spec/fixtures/controlrepos/vendored/Puppetfile +3 -0
  37. data/spec/fixtures/controlrepos/vendored/Puppetfile.cron +5 -0
  38. data/spec/fixtures/controlrepos/vendored/environment.conf +1 -0
  39. data/spec/fixtures/controlrepos/vendored/site-modules/role/manifests/cron.pp +9 -0
  40. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/augeas_core-puppet_agent-7.30.0.json +1 -0
  41. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/augeas_core-puppet_agent-8.6.0.json +1 -0
  42. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/cron_core-puppet_agent-7.30.0.json +1 -0
  43. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/cron_core-puppet_agent-8.6.0.json +1 -0
  44. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/host_core-puppet_agent-7.30.0.json +1 -0
  45. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/host_core-puppet_agent-8.6.0.json +1 -0
  46. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/mount_core-puppet_agent-7.30.0.json +1 -0
  47. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/mount_core-puppet_agent-8.6.0.json +1 -0
  48. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/repo_tree-puppet_agent-7.30.0.json +1 -0
  49. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/repo_tree-puppet_agent-8.6.0.json +1 -0
  50. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/scheduled_task-puppet_agent-7.30.0.json +1 -0
  51. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/scheduled_task-puppet_agent-8.6.0.json +1 -0
  52. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/selinux_core-puppet_agent-7.30.0.json +1 -0
  53. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/selinux_core-puppet_agent-8.6.0.json +1 -0
  54. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/sshkeys_core-puppet_agent-7.30.0.json +1 -0
  55. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/sshkeys_core-puppet_agent-8.6.0.json +1 -0
  56. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/yumrepo_core-puppet_agent-7.30.0.json +1 -0
  57. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/yumrepo_core-puppet_agent-8.6.0.json +1 -0
  58. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zfs_core-puppet_agent-7.30.0.json +1 -0
  59. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zfs_core-puppet_agent-8.6.0.json +1 -0
  60. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zone_core-puppet_agent-7.30.0.json +1 -0
  61. data/spec/fixtures/controlrepos/vendored/spec/vendored_modules/zone_core-puppet_agent-8.6.0.json +1 -0
  62. data/spec/onceover/controlrepo_spec.rb +1 -1
  63. data/templates/factsets_README.md.erb +1 -1
  64. data/templates/pre_conditions_README.md.erb +1 -1
  65. metadata +56 -9
@@ -11,9 +11,8 @@ class Onceover
11
11
  # se or an array
12
12
 
13
13
  if facts.is_a?(Array)
14
- returnval = []
15
- facts.each do |fact|
16
- returnval << self.facts_to_vagrant_box(fact)
14
+ returnval = facts.map do |fact|
15
+ self.facts_to_vagrant_box(fact)
17
16
  end
18
17
  return returnval
19
18
  end
@@ -64,9 +63,8 @@ class Onceover
64
63
  warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
65
64
 
66
65
  if facts.is_a?(Array)
67
- returnval = []
68
- facts.each do |fact|
69
- returnval << self.facts_to_platform(fact)
66
+ returnval = facts.map do |fact|
67
+ self.facts_to_platform(fact)
70
68
  end
71
69
  return returnval
72
70
  end
@@ -131,9 +129,9 @@ class Onceover
131
129
 
132
130
  # Do an r10k deploy
133
131
  r10k_deploy(host, {
134
- :puppetfile => true,
135
- :configfile => '/tmp/r10k.yaml',
136
- })
132
+ :puppetfile => true,
133
+ :configfile => '/tmp/r10k.yaml',
134
+ })
137
135
  end
138
136
 
139
137
  # This actually provisions a node and checks that puppet will be able to run and
@@ -153,7 +151,6 @@ class Onceover
153
151
  opts = {:check_idempotency => true}.merge(opts)
154
152
  opts = {:deploy_controlrepo => true}.merge(opts)
155
153
 
156
-
157
154
  raise "Hosts must be a single host object, not an array" if host.is_a?(Array)
158
155
  raise "Class must be a single Class [String], not an array" unless puppet_class.is_a?(String)
159
156
 
@@ -216,8 +213,8 @@ class Onceover
216
213
  # to refer to each key as either a string or an object
217
214
  current_opts.default_proc = proc do |h, k|
218
215
  case k
219
- when String then sym = k.to_sym; h[sym] if h.key?(sym)
220
- when Symbol then str = k.to_s; h[str] if h.key?(str)
216
+ when String then sym = k.to_sym; h[sym] if h.key?(sym)
217
+ when Symbol then str = k.to_s; h[str] if h.key?(str)
221
218
  end
222
219
  end
223
220
 
@@ -44,6 +44,7 @@ This includes deploying using r10k and running all custom tests.
44
44
  optional nil, :format, 'Which RSpec formatter to use, valid options are: documentation, progress, FailureCollector, OnceoverFormatter. You also specify this multiple times', multiple: true, default: :defaults
45
45
  optional nil, :no_workarounds, 'Disables workarounds that have been added for convenience to get around common RSPec issues such as https://github.com/rodjek/rspec-puppet/issues/665'
46
46
  optional :ff, :fail_fast, 'Abort the run after the first failure'
47
+ optional nil, :auto_vendored, 'Attempt to resolve vendored puppet modules. Ex: puppetlabs/cron_core', default: false
47
48
 
48
49
  run do |opts, args, cmd|
49
50
  repo = Onceover::Controlrepo.new(opts)
@@ -119,7 +119,7 @@ class Onceover
119
119
  @spec_dir = opts[:spec_dir] || File.expand_path('./spec', @root)
120
120
  @facts_dir = opts[:facts_dir] || File.expand_path('factsets', @spec_dir)
121
121
  _facts_dirs = [@facts_dir, File.expand_path('../../factsets', __dir__)]
122
- _facts_files = opts[:facts_files] || _facts_dirs.map{|d| File.join(d, '*.json')}
122
+ _facts_files = opts[:facts_files] || _facts_dirs.map{|d| File.join(d, '*.json')}
123
123
  @facts_files = _facts_files.map{|_path| Dir[_path]}.flatten
124
124
 
125
125
  @nodeset_file = opts[:nodeset_file] || File.expand_path('./spec/acceptance/nodesets/onceover-nodes.yml', @root)
@@ -139,7 +139,6 @@ class Onceover
139
139
  end
140
140
  end
141
141
 
142
-
143
142
  def to_s
144
143
  require 'colored'
145
144
 
@@ -182,9 +181,8 @@ class Onceover
182
181
  end
183
182
 
184
183
  # Get all the classes from all of the manifests
185
- classes = []
186
- code_dirs.each do |dir|
187
- classes << get_classes(dir)
184
+ classes = code_dirs.map do |dir|
185
+ get_classes(dir)
188
186
  end
189
187
  classes.flatten
190
188
  end
@@ -208,9 +206,8 @@ class Onceover
208
206
  raise "Filter param must be a hash" unless filter.is_a?(Hash)
209
207
 
210
208
  all_facts.keep_if do |hash|
211
- matches = []
212
- filter.each do |filter_fact,value|
213
- matches << keypair_is_in_hash(hash,filter_fact,value)
209
+ matches = filter.map do |filter_fact,value|
210
+ keypair_is_in_hash(hash,filter_fact,value)
214
211
  end
215
212
  !matches.include? false
216
213
  end
@@ -231,10 +228,9 @@ class Onceover
231
228
  puppetfile.load!
232
229
 
233
230
  output_array = []
234
- threads = []
235
231
  error_array = []
236
- puppetfile.modules.each do |mod|
237
- threads << Thread.new do
232
+ threads = puppetfile.modules.map do |mod|
233
+ Thread.new do
238
234
  begin
239
235
  row = []
240
236
  logger.debug "Loading data for #{mod.full_name}"
@@ -268,7 +264,7 @@ class Onceover
268
264
  "PatchLevel_minor".green
269
265
  else
270
266
  "No".green
271
- end
267
+ end
272
268
 
273
269
  row << mod.v3_module.endorsement
274
270
  superseded_by = mod.v3_module.superseded_by
@@ -290,8 +286,8 @@ class Onceover
290
286
  error_array << error
291
287
  logger.debug "Error loading module #{mod.full_name} - #{e.inspect}"
292
288
  end
293
- end
294
289
  end
290
+ end
295
291
 
296
292
  threads.map(&:join)
297
293
 
@@ -313,13 +309,12 @@ class Onceover
313
309
  # TODO: Make sure we can deal with :latest
314
310
 
315
311
  # Create threading resources
316
- threads = []
317
- queue = Queue.new
312
+ queue = Queue.new
318
313
  queue.push(puppetfile_string)
319
314
 
320
315
  puppetfile.modules.keep_if {|m| m.is_a?(R10K::Module::Forge)}
321
- puppetfile.modules.each do |mod|
322
- threads << Thread.new do
316
+ threads = puppetfile.modules.map do |mod|
317
+ Thread.new do
323
318
  logger.debug "Getting latest version of #{mod.full_name}"
324
319
  latest_version = mod.v3_module.current_release.version
325
320
 
@@ -360,14 +355,14 @@ class Onceover
360
355
  # Set it up as a symlink, because we are using local files in the Puppetfile
361
356
  symlinks << {
362
357
  'name' => mod.name,
363
- 'dir' => mod.expected_version[:path]
358
+ 'dir' => mod.expected_version[:path]
364
359
  }
365
360
  elsif mod.expected_version.is_a?(String)
366
361
  # Set it up as a normal forge module
367
362
  forge_modules << {
368
363
  'name' => mod.name,
369
364
  'repo' => mod.title,
370
- 'ref' => mod.expected_version
365
+ 'ref' => mod.expected_version
371
366
  }
372
367
  end
373
368
  elsif mod.is_a? R10K::Module::Git
@@ -377,7 +372,7 @@ class Onceover
377
372
  # I know I shouldn't be doing this, but trust me, there are no methods
378
373
  # anywhere that expose this value, I looked.
379
374
  'repo' => mod.instance_variable_get(:@remote),
380
- 'ref' => mod.version
375
+ 'ref' => mod.version
381
376
  }
382
377
  end
383
378
  end
@@ -392,7 +387,7 @@ class Onceover
392
387
  Dir["#{dir}/*"].each do |mod|
393
388
  symlinks << {
394
389
  'name' => File.basename(mod),
395
- 'dir' => Pathname.new(File.expand_path(mod)).relative_path_from(Pathname.new(@root))#File.expand_path(mod)
390
+ 'dir' => Pathname.new(File.expand_path(mod)).relative_path_from(Pathname.new(@root))#File.expand_path(mod)
396
391
  }
397
392
  end
398
393
  end
@@ -574,9 +569,9 @@ class Onceover
574
569
  # Add the resulting info to the hosts hash. This is what the
575
570
  # template will output
576
571
  hosts_hash[node_name] = {
577
- :platform => platform,
578
- :boxname => boxname,
579
- :url => url,
572
+ :platform => platform,
573
+ :boxname => boxname,
574
+ :url => url,
580
575
  :comment_out => comment_out
581
576
  }
582
577
  end
@@ -13,6 +13,10 @@ class Onceover
13
13
  skip_r10k_default = !(File.file?(repo.puppetfile))
14
14
  skip_r10k = opts[:skip_r10k] || skip_r10k_default
15
15
  force = opts[:force] || false
16
+ # Only attempt to resolve vendored modules if configured to do so
17
+ auto_vendored = opts[:auto_vendored] || false
18
+
19
+ require 'onceover/vendored_modules' if auto_vendored
16
20
 
17
21
  if repo.tempdir == nil
18
22
  repo.tempdir = Dir.mktmpdir('r10k')
@@ -63,7 +67,11 @@ class Onceover
63
67
  # add to list of files copied to cache by onceover
64
68
  onceover_manifest << relative_source
65
69
 
66
- if File.directory? source
70
+ if File.symlink?(source)
71
+ # Handle symlinks
72
+ link_target = File.readlink(source) # Get the target of the symlink
73
+ FileUtils.ln_s link_target, target, force: true # Create symlink at target
74
+ elsif File.directory? source
67
75
  Find.prune if excluded_dirs.include? source
68
76
  FileUtils.mkdir target
69
77
  else
@@ -90,7 +98,26 @@ class Onceover
90
98
  if /:control_branch/.match(puppetfile_contents)
91
99
  logger.debug "replacing :control_branch mentions in the Puppetfile with #{git_branch}"
92
100
  new_puppetfile_contents = puppetfile_contents.gsub(":control_branch", "'#{git_branch}'")
93
- File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
101
+ File.write("#{temp_controlrepo}/Puppetfile", new_puppetfile_contents)
102
+ end
103
+
104
+ if auto_vendored
105
+ tmp_puppetfile = File.join(temp_controlrepo, 'Puppetfile')
106
+ tmp_puppetfile_contents = File.read(tmp_puppetfile)
107
+ vm = Onceover::VendoredModules.new({ repo: repo })
108
+ puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: temp_controlrepo)
109
+ vm.puppetfile_missing_vendored(puppetfile)
110
+ unless vm.missing_vendored.empty?
111
+ missing_slugs = vm.missing_vendored.map do |missing_mod|
112
+ missing_mod.keys[0]
113
+ end
114
+ logger.debug "Adding #{missing_slugs} to #{tmp_puppetfile}"
115
+ modlines = vm.missing_vendored.map do |missing_mod|
116
+ mod_slug = missing_mod.keys[0]
117
+ "mod '#{mod_slug}',\n git: '#{missing_mod[mod_slug][:git]}',\n ref: '#{missing_mod[mod_slug][:ref]}'"
118
+ end.join("\n")
119
+ File.write(tmp_puppetfile, "#{tmp_puppetfile_contents}\n# Onceover Managed Vendored Modules\n#{modlines}")
120
+ end
94
121
  end
95
122
  end
96
123
 
@@ -28,10 +28,8 @@ class Onceover
28
28
  @members = []
29
29
  else
30
30
  # Turn it into a full list
31
- member_objects = []
32
-
33
31
  # This should also handle lists that include groups
34
- members.each { |member| member_objects << Onceover::TestConfig.find_list(member) }
32
+ member_objects = members.map { |member| Onceover::TestConfig.find_list(member) }
35
33
  member_objects.flatten!
36
34
 
37
35
  # Check that they are all the same type
@@ -8,8 +8,8 @@ module Onceover::Logger
8
8
  'bright',
9
9
  :levels => {
10
10
  :debug => :cyan,
11
- :info => :green,
12
- :warn => :yellow,
11
+ :info => :green,
12
+ :warn => :yellow,
13
13
  :error => :red,
14
14
  :fatal => [:white, :on_red]
15
15
  }
@@ -18,7 +18,7 @@ module Onceover::Logger
18
18
  Logging.appenders.stdout(
19
19
  'stdout',
20
20
  :layout => Logging.layouts.pattern(
21
- :pattern => '%l\t -> %m\n',
21
+ :pattern => '%l\t -> %m\n',
22
22
  :color_scheme => 'bright'
23
23
  )
24
24
  )
data/lib/onceover/node.rb CHANGED
@@ -4,7 +4,6 @@ class Onceover
4
4
  class Node
5
5
  @@all = []
6
6
 
7
-
8
7
  attr_accessor :name
9
8
  attr_accessor :beaker_node
10
9
  attr_accessor :fact_set
@@ -44,7 +43,6 @@ class Onceover
44
43
  end
45
44
 
46
45
  @@all << self
47
-
48
46
  end
49
47
 
50
48
  def self.find(node_name)
@@ -4,7 +4,6 @@ require 'pathname'
4
4
  @repo = nil
5
5
  @config = nil
6
6
 
7
-
8
7
  desc 'Writes a `fixtures.yml` file based on the Puppetfile'
9
8
  task :generate_fixtures do
10
9
  repo = Onceover::Controlrepo.new
@@ -15,7 +14,6 @@ task :generate_fixtures do
15
14
  File.write(File.expand_path('./.fixtures.yml', repo.root), repo.fixtures)
16
15
  end
17
16
 
18
-
19
17
  desc "Modifies your `hiera.yaml` to point at the hieradata relative to its position."
20
18
  task :hiera_setup do
21
19
  repo = Onceover::Controlrepo.new
@@ -45,7 +43,6 @@ task :generate_onceover_yaml do
45
43
  puts ERB.new(onceover_yaml_template, nil, '-').result(binding)
46
44
  end
47
45
 
48
-
49
46
  task :generate_nodesets do
50
47
  warn "[DEPRECATION] #{__method__} is deprecated due to the removal of Beaker"
51
48
 
@@ -75,7 +72,7 @@ task :generate_nodesets do
75
72
  comment_out = false
76
73
  box_info = MultiJson.load(response.body)
77
74
  box_info['current_version']['providers'].each do |provider|
78
- if provider['name'] == 'virtualbox'
75
+ if provider['name'] == 'virtualbox'
79
76
  url = provider['original_url']
80
77
  end
81
78
  end
@@ -86,5 +83,13 @@ task :generate_nodesets do
86
83
  fixtures_template = File.read(File.expand_path('./nodeset.yaml.erb', template_dir))
87
84
  puts ERB.new(fixtures_template, nil, '-').result(binding)
88
85
  end
86
+ end
87
+
88
+ desc 'Create cache for vendored modules'
89
+ task :generate_vendor_cache do
90
+ require 'onceover/controlrepo'
91
+ require 'onceover/vendored_modules'
89
92
 
93
+ repo = Onceover::Controlrepo.new(debug: true)
94
+ Onceover::VendoredModules.new({ repo: repo, cachedir: File.join(repo.spec_dir, 'vendored_modules'), force_update: true })
90
95
  end
@@ -127,9 +127,9 @@ class OnceoverFormatter
127
127
 
128
128
  scanned_errors.map do |error_matches|
129
129
  {
130
- text: error_matches[0],
131
- file: calculate_relative_source(error_matches[1]),
132
- line: error_matches[2],
130
+ text: error_matches[0],
131
+ file: calculate_relative_source(error_matches[1]),
132
+ line: error_matches[2],
133
133
  column: error_matches[3],
134
134
  }
135
135
  end
@@ -223,7 +223,6 @@ class OnceoverFormatter
223
223
  def longest_group
224
224
  RSpec.configuration.world.example_groups.max { |a,b| a.description.length <=> b.description.length}.description.length
225
225
  end
226
-
227
226
  end
228
227
 
229
228
  class OnceoverFormatterParallel < OnceoverFormatter
data/lib/onceover/test.rb CHANGED
@@ -12,9 +12,8 @@ class Onceover
12
12
  # it will then detect them and expand them out into their respective objects so that
13
13
  # we just end up with a list of nodes and classes
14
14
  def initialize(on_this, test_this, test_config)
15
-
16
15
  @default_test_config = {
17
- 'check_idempotency' => true,
16
+ 'check_idempotency' => true,
18
17
  'runs_before_idempotency' => 1
19
18
  }
20
19
 
@@ -282,9 +282,9 @@ class Onceover
282
282
  def run_filters(tests)
283
283
  # All of this needs to be applied AFTER deduplication but BEFORE writing
284
284
  filters = {
285
- 'tags' => @filter_tags,
285
+ 'tags' => @filter_tags,
286
286
  'classes' => @filter_classes,
287
- 'nodes' => @filter_nodes
287
+ 'nodes' => @filter_nodes
288
288
  }
289
289
  filters.each do |method, filter_list|
290
290
  if filter_list
@@ -0,0 +1,186 @@
1
+ require 'puppet/version'
2
+ require 'net/http'
3
+ require 'uri'
4
+ require 'multi_json'
5
+ require 'r10k/module_loader/puppetfile'
6
+ require 'onceover/logger'
7
+
8
+ ### operations
9
+ #
10
+ # 1. resolve all the component json files in the puppet-agent repo for vendored modules
11
+ # 2. parse each json file and determine vendored modules repo + ref
12
+ #
13
+ ###
14
+
15
+ ## Example
16
+ #
17
+ # vm = Onceover::VendoredModules.new
18
+ # puts vm.vendored_references
19
+ # puppetfile = R10K::ModuleLoader::Puppetfile.new(basedir: '.')
20
+ # vm.puppetfile_missing_vendored(puppetfile)
21
+ # puts vm.missing_vendored.inspect
22
+
23
+ class Onceover
24
+ class VendoredModules
25
+ attr_reader :vendored_references, :missing_vendored
26
+
27
+ def initialize(opts = {})
28
+ @repo = opts[:repo] || Onceover::Controlrepo.new
29
+ @cachedir = opts[:cachedir] || File.join(@repo.tempdir, 'vendored_modules')
30
+ @puppet_version = Gem::Version.new(Puppet.version)
31
+ @puppet_major_version = Gem::Version.new(@puppet_version.segments[0])
32
+ @force_update = opts[:force_update] || false
33
+
34
+ @missing_vendored = []
35
+
36
+ # This only applies to puppet >= 6 so bail early
37
+ raise 'Auto resolving vendored modules only applies to puppet versions >= 6' unless @puppet_major_version >= Gem::Version.new('6')
38
+
39
+ # Create cachedir
40
+ unless File.directory?(@cachedir)
41
+ logger.debug "Creating #{@cachedir}"
42
+ FileUtils.mkdir_p(@cachedir)
43
+ end
44
+
45
+ # Location of user provided caches:
46
+ # control-repo/spec/vendored_modules/<component>-puppet_agent-<agent version>.json
47
+ @manual_vendored_dir = File.join(@repo.spec_dir, 'vendored_modules')
48
+
49
+ # Get the entire file tree of the puppetlabs/puppet-agent repository
50
+ # https://docs.github.com/en/rest/git/trees?apiVersion=2022-11-28#get-a-tree
51
+ puppet_agent_tree = query_or_cache(
52
+ "https://api.github.com/repos/puppetlabs/puppet-agent/git/trees/#{@puppet_version}",
53
+ { recursive: true },
54
+ component_cache('repo_tree'),
55
+ )
56
+ # Get only the module-puppetlabs-<something>_core.json component files
57
+ vendored_components = puppet_agent_tree['tree'].select { |file| %r{configs/components/module-puppetlabs-\w+\.json}.match(file['path']) }
58
+ # Get the contents of each component file
59
+ # https://docs.github.com/en/rest/git/blobs?apiVersion=2022-11-28#get-a-blob
60
+ @vendored_references = vendored_components.map do |component|
61
+ mod_slug = component['path'].match(/.*(puppetlabs-\w+).json$/)[1]
62
+ mod_name = mod_slug.match(/puppetlabs-(\w+)/)[1]
63
+ query_or_cache(
64
+ component['url'],
65
+ nil,
66
+ component_cache(mod_name),
67
+ )
68
+ end
69
+ end
70
+
71
+ def component_cache(component)
72
+ # Ideally we want a cache for the version of the puppet agent used in tests
73
+ desired_name = "#{component}-puppet_agent-#{@puppet_version}.json"
74
+ # By default look for any caches created during previous runs
75
+ cache_file = File.join(@cachedir, desired_name)
76
+
77
+ # If the user provides their own cache
78
+ if !@force_update && File.directory?(@manual_vendored_dir)
79
+ # Check for any '<component>-puppet_agent-<puppet version>.json' files
80
+ dg = Dir.glob(File.join(@manual_vendored_dir, "#{component}-puppet_agent*"))
81
+ # Check if there are multiple versions of the component cache
82
+ if dg.size > 1
83
+ # If there is the same version supplied as whats being tested against use that
84
+ if dg.any? { |s| s[desired_name] }
85
+ cache_file = File.join(@manual_vendored_dir, desired_name)
86
+ # If there are any with the same major version, use the latest supplied
87
+ elsif dg.any? { |s| s["#{component}-puppet_agent-#{@puppet_major_version}"] }
88
+ maj_match = dg.select { |f| /#{component}-puppet_agent-#{@puppet_major_version}.\d+\.\d+\.json/.match(f) }
89
+ maj_match.each do |f|
90
+ next unless (version_from_file(cache_file) == version_from_file(desired_name)) || (version_from_file(f) >= version_from_file(cache_file))
91
+
92
+ # if the current cache version matches the desired version, use the first matching major version in user cache
93
+ # if there are multiple major version matches in user cache, use the latest
94
+ cache_file = f
95
+ end
96
+ # Otherwise just use the latest supplied
97
+ else
98
+ dg.each { |f| cache_file = f if version_from_file(f) >= version_from_file(cache_file) }
99
+ end
100
+ # If there is only one use that
101
+ elsif dg.size == 1
102
+ cache_file = dg[0]
103
+ end
104
+ end
105
+
106
+ # Warn the user if cached version does not match whats being used to test
107
+ cache_version = version_from_file(cache_file)
108
+ logger.warn "Cache for #{component} is for puppet_agent #{cache_version}, while you are testing against puppet_agent #{@puppet_version}. Consider updating your cache to ensure consistent behavior in your tests" if cache_version != @puppet_version
109
+
110
+ cache_file
111
+ end
112
+
113
+ def version_from_file(cache_file)
114
+ version_regex = /.*-puppet_agent-(\d+\.\d+\.\d+)\.json/
115
+ Gem::Version.new(version_regex.match(cache_file)[1])
116
+ end
117
+
118
+ # Currently expects to be passed a R10K::Puppetfile object.
119
+ # ex: R10K::ModuleLoader::Puppetfile.new(basedir: '.')
120
+ def puppetfile_missing_vendored(puppetfile)
121
+ puppetfile.load
122
+ @vendored_references.each do |mod|
123
+ # Extract name and slug from url
124
+ mod_slug = mod['url'].match(/.*(puppetlabs-\w+)\.git/)[1]
125
+ mod_name = mod_slug.match(/^puppetlabs-(\w+)$/)[1]
126
+ # Array of modules whos names match
127
+ existing = puppetfile.modules.select { |e_mod| e_mod.name == mod_name }
128
+ if existing.empty?
129
+ # Change url to https instead of ssh to allow anonymous git clones
130
+ # so that users do not need to have an ssh keypair associated with a Github account
131
+ url = mod['url'].gsub('git@github.com:', 'https://github.com/')
132
+ @missing_vendored << { mod_slug => { git: url, ref: mod['ref'] } }
133
+ logger.debug "#{mod_name} found to be missing in Puppetfile"
134
+ else
135
+ logger.debug "#{mod_name} found in Puppetfile. Using the specified version"
136
+ end
137
+ end
138
+ end
139
+
140
+ # Return json from a query whom caches, or from the cache to avoid spamming github
141
+ def query_or_cache(url, params, filepath)
142
+ if (File.exist? filepath) && (@force_update == false)
143
+ logger.debug "Using cache: #{filepath}"
144
+ json = read_json_dump(filepath)
145
+ else
146
+ logger.debug "Making GET request to: #{url}"
147
+ json = github_get(url, params)
148
+ logger.debug "Caching response to: #{filepath}"
149
+ write_json_dump(filepath, json)
150
+ end
151
+ json
152
+ end
153
+
154
+ # Given a github url and optional query parameters, return the parsed json body
155
+ def github_get(url, params)
156
+ uri = URI.parse(url)
157
+ uri.query = URI.encode_www_form(params) if params
158
+ http = Net::HTTP.new(uri.host, uri.port)
159
+ http.use_ssl = (uri.scheme == 'https')
160
+ request = Net::HTTP::Get.new(uri.request_uri)
161
+ request['Accept'] = 'application/vnd.github.raw+json'
162
+ request['X-GitHub-Api-Version'] = '2022-11-28'
163
+ response = http.request(request)
164
+
165
+ case response
166
+ when Net::HTTPOK # 200
167
+ MultiJson.load(response.body)
168
+ else
169
+ # Expose the ratelimit response headers
170
+ # https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api?apiVersion=2022-11-28#checking-the-status-of-your-rate-limit
171
+ ratelimit_headers = response.to_hash.select { |k, _v| k =~ /x-ratelimit.*/ }
172
+ raise "#{response.code} #{response.message} #{ratelimit_headers}"
173
+ end
174
+ end
175
+
176
+ # Returns parsed json of file
177
+ def read_json_dump(filepath)
178
+ MultiJson.load(File.read(filepath))
179
+ end
180
+
181
+ # Writes json to a file
182
+ def write_json_dump(filepath, json_data)
183
+ File.write(filepath, MultiJson.dump(json_data))
184
+ end
185
+ end
186
+ end
data/onceover.gemspec CHANGED
@@ -4,10 +4,10 @@ $LOAD_PATH.unshift File.expand_path('lib', __dir__)
4
4
 
5
5
  Gem::Specification.new do |s| # rubocop:disable Gemspec/RequireMFA
6
6
  s.name = "onceover"
7
- s.version = "3.22.0"
8
- s.authors = ["Dylan Ratcliffe"]
9
- s.email = ["dylan.ratcliffe@puppet.com"]
10
- s.homepage = "https://github.com/dylanratcliffe/onceover"
7
+ s.version = "4.0.0"
8
+ s.authors = ["Dylan Ratcliffe", 'Vox Pupuli']
9
+ s.email = ["voxpupuli@groups.io"]
10
+ s.homepage = "https://github.com/voxpupuli/onceover"
11
11
  s.summary = "Testing tools for Puppet controlrepos"
12
12
  s.description = "Automatically generates tests for your Puppet code"
13
13
  s.licenses = 'Apache-2.0'
@@ -17,22 +17,25 @@ Gem::Specification.new do |s| # rubocop:disable Gemspec/RequireMFA
17
17
  s.bindir = 'bin'
18
18
  s.executables = 'onceover'
19
19
 
20
+ s.required_ruby_version = Gem::Requirement.new('>= 2.7')
21
+
20
22
  # Runtime dependencies, but also probably dependencies of requiring projects
21
- s.add_runtime_dependency 'backticks', '>= 1.0.2'
22
- s.add_runtime_dependency 'colored', '>= 1.2'
23
- s.add_runtime_dependency 'cri', '>= 2.6'
24
- s.add_runtime_dependency 'deep_merge', '>= 1.0.0'
25
- s.add_runtime_dependency 'git'
26
- s.add_runtime_dependency 'logging', '>= 2.0.0'
27
- s.add_runtime_dependency 'multi_json', '>= 1.10'
28
- s.add_runtime_dependency 'parallel_tests', ">= 2.0.0"
29
- s.add_runtime_dependency 'puppet', '>=4.0'
30
- s.add_runtime_dependency 'puppetlabs_spec_helper', ">= 0.4.0"
31
- s.add_runtime_dependency 'r10k', '>=2.1.0'
32
- s.add_runtime_dependency 'rake', '>= 10.0.0'
33
- s.add_runtime_dependency 'rspec', '>= 3.0.0'
34
- s.add_runtime_dependency 'rspec_junit_formatter', '>= 0.2.0'
35
- s.add_runtime_dependency 'rspec-puppet', ">= 2.4.0"
36
- s.add_runtime_dependency 'terminal-table', '>= 1.8.0'
37
- s.add_runtime_dependency 'versionomy', '>= 0.5.0'
23
+ s.add_dependency 'backticks', '>= 1.0.2'
24
+ s.add_dependency 'colored', '>= 1.2'
25
+ s.add_dependency 'cri', '>= 2.6'
26
+ s.add_dependency 'deep_merge', '>= 1.0.0'
27
+ s.add_dependency 'git'
28
+ s.add_dependency 'logging', '>= 2.0.0'
29
+ s.add_dependency 'multi_json', '>= 1.10'
30
+ s.add_dependency 'parallel_tests', ">= 2.0.0"
31
+ s.add_dependency 'puppet', '>=4.0'
32
+ s.add_dependency 'puppetlabs_spec_helper', ">= 0.4.0"
33
+ s.add_dependency 'r10k', '>=2.1.0'
34
+ s.add_dependency 'rake', '>= 10.0.0'
35
+ s.add_dependency 'rspec', '>= 3.0.0'
36
+ s.add_dependency 'rspec_junit_formatter', '>= 0.2.0'
37
+ s.add_dependency 'rspec-puppet', ">= 2.4.0"
38
+ s.add_dependency 'terminal-table', '>= 1.8.0'
39
+ s.add_dependency 'versionomy', '>= 0.5.0'
40
+ s.add_development_dependency 'voxpupuli-rubocop', '~> 3.0.0'
38
41
  end
@@ -4,4 +4,4 @@ This directory is where we put any custom factsets that we want to use. They can
4
4
 
5
5
  **Hot tip:** If you already have factsets in here when you run `onceover init` they will be picked up and added to the config file Automatically
6
6
 
7
- More info: https://github.com/dylanratcliffe/onceover#factsets
7
+ More info: https://github.com/voxpupuli/onceover#factsets
@@ -21,4 +21,4 @@ service { 'pe-puppetserver':
21
21
 
22
22
  This will mean that the `pe-puppetserver` service is in the catalog for spec testing and will even allow you to try to restart it during acceptance tests without the service actually being present.
23
23
 
24
- More info: https://github.com/dylanratcliffe/onceover#using-workarounds
24
+ More info: https://github.com/voxpupuli/onceover#using-workarounds
@@ -4,4 +4,4 @@ This directory is where we put any custom factsets that we want to use. They can
4
4
 
5
5
  **Hot tip:** If you already have factsets in here when you run `onceover init` they will be picked up and added to the config file Automatically
6
6
 
7
- More info: https://github.com/dylanratcliffe/onceover#factsets
7
+ More info: https://github.com/voxpupuli/onceover#factsets