r10k 3.9.3 → 3.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/rspec_tests.yml +1 -1
  3. data/CHANGELOG.mkd +9 -0
  4. data/doc/dynamic-environments/configuration.mkd +7 -0
  5. data/integration/Rakefile +1 -1
  6. data/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb +3 -9
  7. data/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb +8 -14
  8. data/lib/r10k/action/deploy/environment.rb +3 -0
  9. data/lib/r10k/action/runner.rb +11 -6
  10. data/lib/r10k/git/cache.rb +1 -1
  11. data/lib/r10k/initializers.rb +7 -0
  12. data/lib/r10k/module/forge.rb +5 -1
  13. data/lib/r10k/module_loader/puppetfile.rb +195 -0
  14. data/lib/r10k/module_loader/puppetfile/dsl.rb +37 -0
  15. data/lib/r10k/puppetfile.rb +77 -153
  16. data/lib/r10k/settings.rb +3 -0
  17. data/lib/r10k/source/base.rb +10 -0
  18. data/lib/r10k/source/git.rb +5 -0
  19. data/lib/r10k/source/svn.rb +4 -0
  20. data/lib/r10k/util/purgeable.rb +70 -8
  21. data/lib/r10k/version.rb +1 -1
  22. data/locales/r10k.pot +37 -33
  23. data/spec/fixtures/unit/action/r10k_forge_auth.yaml +4 -0
  24. data/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml +3 -0
  25. data/spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/managed_subdir_2/ignored_1 +0 -0
  26. data/spec/fixtures/unit/util/purgeable/managed_two/.hidden/unmanaged_3 +0 -0
  27. data/spec/unit/action/deploy/environment_spec.rb +9 -0
  28. data/spec/unit/action/deploy/module_spec.rb +38 -14
  29. data/spec/unit/action/runner_spec.rb +49 -25
  30. data/spec/unit/git/cache_spec.rb +14 -0
  31. data/spec/unit/module/forge_spec.rb +8 -1
  32. data/spec/unit/module_loader/puppetfile_spec.rb +330 -0
  33. data/spec/unit/puppetfile_spec.rb +99 -193
  34. data/spec/unit/settings_spec.rb +6 -2
  35. data/spec/unit/util/purgeable_spec.rb +38 -6
  36. metadata +10 -3
@@ -4,10 +4,17 @@ require 'r10k/module'
4
4
  require 'r10k/util/purgeable'
5
5
  require 'r10k/errors'
6
6
  require 'r10k/content_synchronizer'
7
+ require 'r10k/module_loader/puppetfile/dsl'
8
+ require 'r10k/module_loader/puppetfile'
7
9
 
8
10
  module R10K
11
+
12
+ # Deprecated, use R10K::ModuleLoader::Puppetfile#load to load content,
13
+ # provide the `:modules` key of the returned Hash to
14
+ # R10K::ContentSynchronizer (either the `serial_sync` or `concurrent_sync`)
15
+ # and the remaining keys (`:managed_directories`, `:desired_contents`, and
16
+ # `:purge_exclusions`) to R10K::Util::Cleaner.
9
17
  class Puppetfile
10
- # Defines the data members of a Puppetfile
11
18
 
12
19
  include R10K::Settings::Mixin
13
20
 
@@ -19,25 +26,17 @@ class Puppetfile
19
26
  # @return [String] The URL to use for the Puppet Forge
20
27
  attr_reader :forge
21
28
 
22
- # @!attribute [r] modules
23
- # @return [Array<R10K::Module>]
24
- attr_reader :modules
25
-
26
29
  # @!attribute [r] basedir
27
30
  # @return [String] The base directory that contains the Puppetfile
28
31
  attr_reader :basedir
29
32
 
30
- # @!attribute [r] moduledir
31
- # @return [String] The directory to install the modules #{basedir}/modules
32
- attr_reader :moduledir
33
-
34
33
  # @!attrbute [r] puppetfile_path
35
34
  # @return [String] The path to the Puppetfile
36
35
  attr_reader :puppetfile_path
37
36
 
38
- # @!attribute [rw] environment
37
+ # @!attribute [r] environment
39
38
  # @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to.
40
- attr_accessor :environment
39
+ attr_reader :environment
41
40
 
42
41
  # @!attribute [rw] force
43
42
  # @return [Boolean] Overwrite any locally made changes
@@ -47,6 +46,10 @@ class Puppetfile
47
46
  # @return [Hash] Various settings overridden from normal configs
48
47
  attr_reader :overrides
49
48
 
49
+ # @!attribute [r] loader
50
+ # @return [R10K::ModuleLoader::Puppetfile] The internal module loader
51
+ attr_reader :loader
52
+
50
53
  # @param [String] basedir
51
54
  # @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options.
52
55
  # Usage as moduledir is deprecated. Only use as options, defaults to nil
@@ -67,142 +70,118 @@ class Puppetfile
67
70
  @moduledir = deprecated_moduledir_arg || options.delete(:moduledir) || File.join(basedir, 'modules')
68
71
  @puppetfile_name = deprecated_name_arg || options.delete(:puppetfile_name) || 'Puppetfile'
69
72
  @puppetfile_path = deprecated_path_arg || options.delete(:puppetfile_path) || File.join(basedir, @puppetfile_name)
73
+ @environment = options.delete(:environment)
70
74
 
71
75
  @overrides = options.delete(:overrides) || {}
76
+ @default_branch_override = @overrides.dig(:environments, :default_branch_override)
72
77
 
73
78
  logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
74
79
 
75
- @modules = []
76
- @managed_content = {}
77
80
  @forge = 'forgeapi.puppetlabs.com'
78
81
 
82
+ @loader = ::R10K::ModuleLoader::Puppetfile.new(
83
+ basedir: @basedir,
84
+ moduledir: @moduledir,
85
+ puppetfile: @puppetfile_path,
86
+ forge: @forge,
87
+ overrides: @overrides,
88
+ environment: @environment
89
+ )
90
+
91
+ @loaded_content = {
92
+ modules: [],
93
+ managed_directories: [],
94
+ desired_contents: [],
95
+ purge_exclusions: []
96
+ }
97
+
79
98
  @loaded = false
80
99
  end
81
100
 
101
+ # @param [String] default_branch_override The default branch to use
102
+ # instead of one specified in the module declaration, if applicable.
103
+ # Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass
104
+ # the default_branch_override as an option on initialization.
82
105
  def load(default_branch_override = nil)
83
- return true if self.loaded?
84
- if File.readable? @puppetfile_path
85
- self.load!(default_branch_override)
106
+ if self.loaded?
107
+ return @loaded_content
86
108
  else
87
- logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect}
109
+ if !File.readable?(@puppetfile_path)
110
+ logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect}
111
+ else
112
+ self.load!(default_branch_override)
113
+ end
88
114
  end
89
115
  end
90
116
 
117
+ # @param [String] default_branch_override The default branch to use
118
+ # instead of one specified in the module declaration, if applicable.
119
+ # Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass
120
+ # the default_branch_override as an option on initialization.
91
121
  def load!(default_branch_override = nil)
92
- @default_branch_override = default_branch_override
93
122
 
94
- dsl = R10K::Puppetfile::DSL.new(self)
95
- dsl.instance_eval(puppetfile_contents, @puppetfile_path)
123
+ if default_branch_override && (default_branch_override != @default_branch_override)
124
+ logger.warn("Mismatch between passed and initialized default branch overrides, preferring passed value.")
125
+ @loader.default_branch_override = default_branch_override
126
+ end
96
127
 
97
- validate_no_duplicate_names(@modules)
128
+ @loaded_content = @loader.load
98
129
  @loaded = true
99
- rescue SyntaxError, LoadError, ArgumentError, NameError => e
100
- raise R10K::Error.wrap(e, _("Failed to evaluate %{path}") % {path: @puppetfile_path})
130
+
131
+ @loaded_content
101
132
  end
102
133
 
103
134
  def loaded?
104
135
  @loaded
105
136
  end
106
137
 
107
- # @param [Array<String>] modules
108
- def validate_no_duplicate_names(modules)
109
- dupes = modules
110
- .group_by { |mod| mod.name }
111
- .select { |_, v| v.size > 1 }
112
- .map(&:first)
113
- unless dupes.empty?
114
- msg = _('Puppetfiles cannot contain duplicate module names.')
115
- msg += ' '
116
- msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') })
117
- raise R10K::Error.new(msg)
118
- end
138
+ def modules
139
+ @loaded_content[:modules]
119
140
  end
120
141
 
121
- # @param [String] forge
122
- def set_forge(forge)
123
- @forge = forge
142
+ # @see R10K::ModuleLoader::Puppetfile#add_module for upcoming signature changes
143
+ def add_module(name, args)
144
+ @loader.add_module(name, args)
124
145
  end
125
146
 
126
- # @param [String] moduledir
127
- def set_moduledir(moduledir)
128
- @moduledir = if Pathname.new(moduledir).absolute?
129
- moduledir
130
- else
131
- File.join(basedir, moduledir)
132
- end
147
+ def set_moduledir(dir)
148
+ @loader.set_moduledir(dir)
133
149
  end
134
150
 
135
- # @param [String] name
136
- # @param [Hash, String, Symbol] args Calling with anything but a Hash is
137
- # deprecated. The DSL will now convert String and Symbol versions to
138
- # Hashes of the shape
139
- # { version: <String or Symbol> }
140
- #
141
- def add_module(name, args)
142
- if !args.is_a?(Hash)
143
- args = { version: args }
144
- end
145
-
146
- args[:overrides] = @overrides
147
-
148
- if install_path = args.delete(:install_path)
149
- install_path = resolve_install_path(install_path)
150
- validate_install_path(install_path, name)
151
- else
152
- install_path = @moduledir
153
- end
154
-
155
- if @default_branch_override != nil
156
- args[:default_branch_override] = @default_branch_override
157
- end
158
-
159
-
160
- mod = R10K::Module.new(name, install_path, args, @environment)
161
- mod.origin = :puppetfile
162
-
163
- # Do not load modules if they would conflict with the attached
164
- # environment
165
- if environment && environment.module_conflicts?(mod)
166
- mod = nil
167
- return @modules
168
- end
151
+ def set_forge(forge)
152
+ @loader.set_forge(forge)
153
+ end
169
154
 
170
- # Keep track of all the content this Puppetfile is managing to enable purging.
171
- @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
172
- @managed_content[install_path] << mod.name
155
+ def moduledir
156
+ @loader.moduledir
157
+ end
173
158
 
174
- @modules << mod
159
+ def environment=(env)
160
+ @loader.environment = env
161
+ @environment = env
175
162
  end
176
163
 
177
164
  include R10K::Util::Purgeable
178
165
 
179
166
  def managed_directories
180
- self.load unless @loaded
167
+ self.load
181
168
 
182
- dirs = @managed_content.keys
183
- dirs.delete(real_basedir)
184
- dirs
169
+ @loaded_content[:managed_directories]
185
170
  end
186
171
 
187
172
  # Returns an array of the full paths to all the content being managed.
188
173
  # @note This implements a required method for the Purgeable mixin
189
174
  # @return [Array<String>]
190
175
  def desired_contents
191
- self.load unless @loaded
176
+ self.load
192
177
 
193
- @managed_content.flat_map do |install_path, modnames|
194
- modnames.collect { |name| File.join(install_path, name) }
195
- end
178
+ @loaded_content[:desired_contents]
196
179
  end
197
180
 
198
181
  def purge_exclusions
199
- exclusions = managed_directories
200
-
201
- if environment && environment.respond_to?(:desired_contents)
202
- exclusions += environment.desired_contents
203
- end
182
+ self.load
204
183
 
205
- exclusions
184
+ @loaded_content[:purge_exclusions]
206
185
  end
207
186
 
208
187
  def accept(visitor)
@@ -225,65 +204,10 @@ class Puppetfile
225
204
 
226
205
  private
227
206
 
228
- def puppetfile_contents
229
- File.read(@puppetfile_path)
230
- end
231
-
232
- def resolve_install_path(path)
233
- pn = Pathname.new(path)
234
-
235
- unless pn.absolute?
236
- pn = Pathname.new(File.join(basedir, path))
237
- end
238
-
239
- # .cleanpath is as good as we can do without touching the filesystem.
240
- # The .realpath methods will also choke if some of the intermediate
241
- # paths are missing, even though we will create them later as needed.
242
- pn.cleanpath.to_s
243
- end
244
-
245
- def validate_install_path(path, modname)
246
- unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
247
- raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
248
- end
249
-
250
- true
251
- end
252
-
253
207
  def real_basedir
254
208
  Pathname.new(basedir).cleanpath.to_s
255
209
  end
256
210
 
257
- class DSL
258
- # A barebones implementation of the Puppetfile DSL
259
- #
260
- # @api private
261
-
262
- def initialize(librarian)
263
- @librarian = librarian
264
- end
265
-
266
- def mod(name, args = nil)
267
- if args.is_a?(Hash)
268
- opts = args
269
- else
270
- opts = { version: args }
271
- end
272
-
273
- @librarian.add_module(name, opts)
274
- end
275
-
276
- def forge(location)
277
- @librarian.set_forge(location)
278
- end
279
-
280
- def moduledir(location)
281
- @librarian.set_moduledir(location)
282
- end
283
-
284
- def method_missing(method, *args)
285
- raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method}
286
- end
287
- end
211
+ DSL = R10K::ModuleLoader::Puppetfile::DSL
288
212
  end
289
213
  end
data/lib/r10k/settings.rb CHANGED
@@ -92,6 +92,9 @@ module R10K
92
92
  URIDefinition.new(:baseurl, {
93
93
  :desc => "The URL to the Puppet Forge to use for downloading modules."
94
94
  }),
95
+ Definition.new(:authorization_token, {
96
+ :desc => "The token for Puppet Forge authorization. Leave blank for unauthorized or license-based connections."
97
+ })
95
98
  ])
96
99
  end
97
100
 
@@ -63,6 +63,16 @@ class R10K::Source::Base
63
63
 
64
64
  end
65
65
 
66
+ # Perform actions to reload environments after the `preload!`. Similar
67
+ # to preload!, and likely to include network queries and rerunning
68
+ # environment generation.
69
+ #
70
+ # @api public
71
+ # @abstract
72
+ # @return [void]
73
+ def reload!
74
+ end
75
+
66
76
  # Enumerate the environments associated with this SVN source.
67
77
  #
68
78
  # @api public
@@ -93,6 +93,11 @@ class R10K::Source::Git < R10K::Source::Base
93
93
  end
94
94
  end
95
95
 
96
+ def reload!
97
+ @cache.sync!
98
+ @environments = generate_environments()
99
+ end
100
+
96
101
  def generate_environments
97
102
  envs = []
98
103
  branch_names.each do |bn|
@@ -65,6 +65,10 @@ class R10K::Source::SVN < R10K::Source::Base
65
65
  @ignore_branch_prefixes = options[:ignore_branch_prefixes]
66
66
  end
67
67
 
68
+ def reload!
69
+ @environments = generate_environments()
70
+ end
71
+
68
72
  # Enumerate the environments associated with this SVN source.
69
73
  #
70
74
  # @return [Array<R10K::Environment::SVN>] An array of environments created
@@ -9,6 +9,12 @@ module R10K
9
9
  # {#desired_contents}
10
10
  module Purgeable
11
11
 
12
+ HIDDEN_FILE = /\.[^.]+/
13
+
14
+ FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH
15
+
16
+ # @deprecated
17
+ #
12
18
  # @!method logger
13
19
  # @abstract Including classes must provide a logger method
14
20
  # @return [Log4r::Logger]
@@ -38,23 +44,79 @@ module R10K
38
44
  end
39
45
  end
40
46
 
47
+ # @deprecated Unused helper function
48
+ #
41
49
  # @return [Array<String>] Directory contents that are expected but not present
42
50
  def pending_contents(recurse)
43
51
  desired_contents - current_contents(recurse)
44
52
  end
45
53
 
54
+ def matches?(test, path)
55
+ if test == path
56
+ true
57
+ elsif File.fnmatch?(test, path, FN_MATCH_OPTS)
58
+ true
59
+ else
60
+ false
61
+ end
62
+ end
63
+
64
+ # A method to collect potentially purgeable content without searching into
65
+ # ignored directories when recursively searching.
66
+ #
67
+ # @param dir [String, Pathname] The directory to search for purgeable content
68
+ # @param exclusion_gobs [Array<String>] A list of file paths or File globs
69
+ # to exclude from recursion (These are generated by the classes that
70
+ # mix this module into them and are typically programatically generated)
71
+ # @param allowed_gobs [Array<String>] A list of file paths or File globs to exclude
72
+ # from recursion (These are passed in by the caller of purge! and typically
73
+ # are user supplied configuration values)
74
+ # @param desireds_not_to_recurse_into [Array<String>] A list of file paths not to
75
+ # recurse into. These are programatically generated, these exist to maintain
76
+ # backwards compatibility with previous implementations that used File.globs
77
+ # for "recursion", ie "**/{*,.[^.]*}" which would not recurse into dot directories.
78
+ # @param recurse [Boolean] Whether or not to recurse into child directories that do
79
+ # not match other filters.
80
+ #
81
+ # @return [Array<String>] Contents which may be purged.
82
+ def potentially_purgeable(dir, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
83
+ children = Pathname.new(dir).children.reject do |path|
84
+ path = path.to_s
85
+
86
+ if exclusion_match = exclusion_globs.find { |exclusion| matches?(exclusion, path) }
87
+ logger.debug2 _("Not purging %{path} due to internal exclusion match: %{exclusion_match}") % {path: path, exclusion_match: exclusion_match}
88
+ elsif allowlist_match = allowed_globs.find { |allowed| matches?(allowed, path) }
89
+ logger.debug _("Not purging %{path} due to whitelist match: %{allowlist_match}") % {path: path, allowlist_match: allowlist_match}
90
+ else
91
+ desired_match = desireds_not_to_recurse_into.grep(path).first
92
+ end
93
+
94
+ !!exclusion_match || !!allowlist_match || !!desired_match
95
+ end
96
+
97
+ children.flat_map do |child|
98
+ if File.directory?(child) && recurse
99
+ potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
100
+ else
101
+ child.to_s
102
+ end
103
+ end
104
+ end
105
+
46
106
  # @return [Array<String>] Directory contents that are present but not expected
47
107
  def stale_contents(recurse, exclusions, whitelist)
48
- fn_match_opts = File::FNM_PATHNAME | File::FNM_DOTMATCH
108
+ dirs = self.managed_directories
109
+ desireds = self.desired_contents
110
+ hidden_desireds, regular_desireds = desireds.partition do |desired|
111
+ HIDDEN_FILE.match(File.basename(desired))
112
+ end
49
113
 
50
- (current_contents(recurse) - desired_contents).reject do |item|
51
- if exclusion_match = exclusions.find { |ex_item| (ex_item == item) || File.fnmatch?(ex_item, item, fn_match_opts) }
52
- logger.debug2 _("Not purging %{item} due to internal exclusion match: %{exclusion_match}") % {item: item, exclusion_match: exclusion_match}
53
- elsif whitelist_match = whitelist.find { |wl_item| (wl_item == item) || File.fnmatch?(wl_item, item, fn_match_opts) }
54
- logger.debug _("Not purging %{item} due to whitelist match: %{whitelist_match}") % {item: item, whitelist_match: whitelist_match}
55
- end
114
+ initial_purgelist = dirs.flat_map do |dir|
115
+ potentially_purgeable(dir, exclusions, whitelist, hidden_desireds, recurse)
116
+ end
56
117
 
57
- !!exclusion_match || !!whitelist_match
118
+ initial_purgelist.reject do |path|
119
+ regular_desireds.any? { |desired| matches?(desired, path) }
58
120
  end
59
121
  end
60
122