r10k 3.9.3 → 3.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/rspec_tests.yml +1 -1
- data/CHANGELOG.mkd +9 -0
- data/doc/dynamic-environments/configuration.mkd +7 -0
- data/integration/Rakefile +1 -1
- data/integration/tests/user_scenario/basic_workflow/negative/neg_specify_deleted_forge_module.rb +3 -9
- data/integration/tests/user_scenario/basic_workflow/single_env_purge_unmanaged_modules.rb +8 -14
- data/lib/r10k/action/deploy/environment.rb +3 -0
- data/lib/r10k/action/runner.rb +11 -6
- data/lib/r10k/git/cache.rb +1 -1
- data/lib/r10k/initializers.rb +7 -0
- data/lib/r10k/module/forge.rb +5 -1
- data/lib/r10k/module_loader/puppetfile.rb +195 -0
- data/lib/r10k/module_loader/puppetfile/dsl.rb +37 -0
- data/lib/r10k/puppetfile.rb +77 -153
- data/lib/r10k/settings.rb +3 -0
- data/lib/r10k/source/base.rb +10 -0
- data/lib/r10k/source/git.rb +5 -0
- data/lib/r10k/source/svn.rb +4 -0
- data/lib/r10k/util/purgeable.rb +70 -8
- data/lib/r10k/version.rb +1 -1
- data/locales/r10k.pot +37 -33
- data/spec/fixtures/unit/action/r10k_forge_auth.yaml +4 -0
- data/spec/fixtures/unit/action/r10k_forge_auth_no_url.yaml +3 -0
- data/spec/fixtures/unit/util/purgeable/managed_one/managed_subdir_1/managed_subdir_2/ignored_1 +0 -0
- data/spec/fixtures/unit/util/purgeable/managed_two/.hidden/unmanaged_3 +0 -0
- data/spec/unit/action/deploy/environment_spec.rb +9 -0
- data/spec/unit/action/deploy/module_spec.rb +38 -14
- data/spec/unit/action/runner_spec.rb +49 -25
- data/spec/unit/git/cache_spec.rb +14 -0
- data/spec/unit/module/forge_spec.rb +8 -1
- data/spec/unit/module_loader/puppetfile_spec.rb +330 -0
- data/spec/unit/puppetfile_spec.rb +99 -193
- data/spec/unit/settings_spec.rb +6 -2
- data/spec/unit/util/purgeable_spec.rb +38 -6
- metadata +10 -3
data/lib/r10k/puppetfile.rb
CHANGED
@@ -4,10 +4,17 @@ require 'r10k/module'
|
|
4
4
|
require 'r10k/util/purgeable'
|
5
5
|
require 'r10k/errors'
|
6
6
|
require 'r10k/content_synchronizer'
|
7
|
+
require 'r10k/module_loader/puppetfile/dsl'
|
8
|
+
require 'r10k/module_loader/puppetfile'
|
7
9
|
|
8
10
|
module R10K
|
11
|
+
|
12
|
+
# Deprecated, use R10K::ModuleLoader::Puppetfile#load to load content,
|
13
|
+
# provide the `:modules` key of the returned Hash to
|
14
|
+
# R10K::ContentSynchronizer (either the `serial_sync` or `concurrent_sync`)
|
15
|
+
# and the remaining keys (`:managed_directories`, `:desired_contents`, and
|
16
|
+
# `:purge_exclusions`) to R10K::Util::Cleaner.
|
9
17
|
class Puppetfile
|
10
|
-
# Defines the data members of a Puppetfile
|
11
18
|
|
12
19
|
include R10K::Settings::Mixin
|
13
20
|
|
@@ -19,25 +26,17 @@ class Puppetfile
|
|
19
26
|
# @return [String] The URL to use for the Puppet Forge
|
20
27
|
attr_reader :forge
|
21
28
|
|
22
|
-
# @!attribute [r] modules
|
23
|
-
# @return [Array<R10K::Module>]
|
24
|
-
attr_reader :modules
|
25
|
-
|
26
29
|
# @!attribute [r] basedir
|
27
30
|
# @return [String] The base directory that contains the Puppetfile
|
28
31
|
attr_reader :basedir
|
29
32
|
|
30
|
-
# @!attribute [r] moduledir
|
31
|
-
# @return [String] The directory to install the modules #{basedir}/modules
|
32
|
-
attr_reader :moduledir
|
33
|
-
|
34
33
|
# @!attrbute [r] puppetfile_path
|
35
34
|
# @return [String] The path to the Puppetfile
|
36
35
|
attr_reader :puppetfile_path
|
37
36
|
|
38
|
-
# @!attribute [
|
37
|
+
# @!attribute [r] environment
|
39
38
|
# @return [R10K::Environment] Optional R10K::Environment that this Puppetfile belongs to.
|
40
|
-
|
39
|
+
attr_reader :environment
|
41
40
|
|
42
41
|
# @!attribute [rw] force
|
43
42
|
# @return [Boolean] Overwrite any locally made changes
|
@@ -47,6 +46,10 @@ class Puppetfile
|
|
47
46
|
# @return [Hash] Various settings overridden from normal configs
|
48
47
|
attr_reader :overrides
|
49
48
|
|
49
|
+
# @!attribute [r] loader
|
50
|
+
# @return [R10K::ModuleLoader::Puppetfile] The internal module loader
|
51
|
+
attr_reader :loader
|
52
|
+
|
50
53
|
# @param [String] basedir
|
51
54
|
# @param [Hash, String, nil] options_or_moduledir The directory to install the modules or a Hash of options.
|
52
55
|
# Usage as moduledir is deprecated. Only use as options, defaults to nil
|
@@ -67,142 +70,118 @@ class Puppetfile
|
|
67
70
|
@moduledir = deprecated_moduledir_arg || options.delete(:moduledir) || File.join(basedir, 'modules')
|
68
71
|
@puppetfile_name = deprecated_name_arg || options.delete(:puppetfile_name) || 'Puppetfile'
|
69
72
|
@puppetfile_path = deprecated_path_arg || options.delete(:puppetfile_path) || File.join(basedir, @puppetfile_name)
|
73
|
+
@environment = options.delete(:environment)
|
70
74
|
|
71
75
|
@overrides = options.delete(:overrides) || {}
|
76
|
+
@default_branch_override = @overrides.dig(:environments, :default_branch_override)
|
72
77
|
|
73
78
|
logger.info _("Using Puppetfile '%{puppetfile}'") % {puppetfile: @puppetfile_path}
|
74
79
|
|
75
|
-
@modules = []
|
76
|
-
@managed_content = {}
|
77
80
|
@forge = 'forgeapi.puppetlabs.com'
|
78
81
|
|
82
|
+
@loader = ::R10K::ModuleLoader::Puppetfile.new(
|
83
|
+
basedir: @basedir,
|
84
|
+
moduledir: @moduledir,
|
85
|
+
puppetfile: @puppetfile_path,
|
86
|
+
forge: @forge,
|
87
|
+
overrides: @overrides,
|
88
|
+
environment: @environment
|
89
|
+
)
|
90
|
+
|
91
|
+
@loaded_content = {
|
92
|
+
modules: [],
|
93
|
+
managed_directories: [],
|
94
|
+
desired_contents: [],
|
95
|
+
purge_exclusions: []
|
96
|
+
}
|
97
|
+
|
79
98
|
@loaded = false
|
80
99
|
end
|
81
100
|
|
101
|
+
# @param [String] default_branch_override The default branch to use
|
102
|
+
# instead of one specified in the module declaration, if applicable.
|
103
|
+
# Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass
|
104
|
+
# the default_branch_override as an option on initialization.
|
82
105
|
def load(default_branch_override = nil)
|
83
|
-
|
84
|
-
|
85
|
-
self.load!(default_branch_override)
|
106
|
+
if self.loaded?
|
107
|
+
return @loaded_content
|
86
108
|
else
|
87
|
-
|
109
|
+
if !File.readable?(@puppetfile_path)
|
110
|
+
logger.debug _("Puppetfile %{path} missing or unreadable") % {path: @puppetfile_path.inspect}
|
111
|
+
else
|
112
|
+
self.load!(default_branch_override)
|
113
|
+
end
|
88
114
|
end
|
89
115
|
end
|
90
116
|
|
117
|
+
# @param [String] default_branch_override The default branch to use
|
118
|
+
# instead of one specified in the module declaration, if applicable.
|
119
|
+
# Deprecated, use R10K::ModuleLoader::Puppetfile directly and pass
|
120
|
+
# the default_branch_override as an option on initialization.
|
91
121
|
def load!(default_branch_override = nil)
|
92
|
-
@default_branch_override = default_branch_override
|
93
122
|
|
94
|
-
|
95
|
-
|
123
|
+
if default_branch_override && (default_branch_override != @default_branch_override)
|
124
|
+
logger.warn("Mismatch between passed and initialized default branch overrides, preferring passed value.")
|
125
|
+
@loader.default_branch_override = default_branch_override
|
126
|
+
end
|
96
127
|
|
97
|
-
|
128
|
+
@loaded_content = @loader.load
|
98
129
|
@loaded = true
|
99
|
-
|
100
|
-
|
130
|
+
|
131
|
+
@loaded_content
|
101
132
|
end
|
102
133
|
|
103
134
|
def loaded?
|
104
135
|
@loaded
|
105
136
|
end
|
106
137
|
|
107
|
-
|
108
|
-
|
109
|
-
dupes = modules
|
110
|
-
.group_by { |mod| mod.name }
|
111
|
-
.select { |_, v| v.size > 1 }
|
112
|
-
.map(&:first)
|
113
|
-
unless dupes.empty?
|
114
|
-
msg = _('Puppetfiles cannot contain duplicate module names.')
|
115
|
-
msg += ' '
|
116
|
-
msg += _("Remove the duplicates of the following modules: %{dupes}" % { dupes: dupes.join(' ') })
|
117
|
-
raise R10K::Error.new(msg)
|
118
|
-
end
|
138
|
+
def modules
|
139
|
+
@loaded_content[:modules]
|
119
140
|
end
|
120
141
|
|
121
|
-
# @
|
122
|
-
def
|
123
|
-
@
|
142
|
+
# @see R10K::ModuleLoader::Puppetfile#add_module for upcoming signature changes
|
143
|
+
def add_module(name, args)
|
144
|
+
@loader.add_module(name, args)
|
124
145
|
end
|
125
146
|
|
126
|
-
|
127
|
-
|
128
|
-
@moduledir = if Pathname.new(moduledir).absolute?
|
129
|
-
moduledir
|
130
|
-
else
|
131
|
-
File.join(basedir, moduledir)
|
132
|
-
end
|
147
|
+
def set_moduledir(dir)
|
148
|
+
@loader.set_moduledir(dir)
|
133
149
|
end
|
134
150
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
# Hashes of the shape
|
139
|
-
# { version: <String or Symbol> }
|
140
|
-
#
|
141
|
-
def add_module(name, args)
|
142
|
-
if !args.is_a?(Hash)
|
143
|
-
args = { version: args }
|
144
|
-
end
|
145
|
-
|
146
|
-
args[:overrides] = @overrides
|
147
|
-
|
148
|
-
if install_path = args.delete(:install_path)
|
149
|
-
install_path = resolve_install_path(install_path)
|
150
|
-
validate_install_path(install_path, name)
|
151
|
-
else
|
152
|
-
install_path = @moduledir
|
153
|
-
end
|
154
|
-
|
155
|
-
if @default_branch_override != nil
|
156
|
-
args[:default_branch_override] = @default_branch_override
|
157
|
-
end
|
158
|
-
|
159
|
-
|
160
|
-
mod = R10K::Module.new(name, install_path, args, @environment)
|
161
|
-
mod.origin = :puppetfile
|
162
|
-
|
163
|
-
# Do not load modules if they would conflict with the attached
|
164
|
-
# environment
|
165
|
-
if environment && environment.module_conflicts?(mod)
|
166
|
-
mod = nil
|
167
|
-
return @modules
|
168
|
-
end
|
151
|
+
def set_forge(forge)
|
152
|
+
@loader.set_forge(forge)
|
153
|
+
end
|
169
154
|
|
170
|
-
|
171
|
-
@
|
172
|
-
|
155
|
+
def moduledir
|
156
|
+
@loader.moduledir
|
157
|
+
end
|
173
158
|
|
174
|
-
|
159
|
+
def environment=(env)
|
160
|
+
@loader.environment = env
|
161
|
+
@environment = env
|
175
162
|
end
|
176
163
|
|
177
164
|
include R10K::Util::Purgeable
|
178
165
|
|
179
166
|
def managed_directories
|
180
|
-
self.load
|
167
|
+
self.load
|
181
168
|
|
182
|
-
|
183
|
-
dirs.delete(real_basedir)
|
184
|
-
dirs
|
169
|
+
@loaded_content[:managed_directories]
|
185
170
|
end
|
186
171
|
|
187
172
|
# Returns an array of the full paths to all the content being managed.
|
188
173
|
# @note This implements a required method for the Purgeable mixin
|
189
174
|
# @return [Array<String>]
|
190
175
|
def desired_contents
|
191
|
-
self.load
|
176
|
+
self.load
|
192
177
|
|
193
|
-
@
|
194
|
-
modnames.collect { |name| File.join(install_path, name) }
|
195
|
-
end
|
178
|
+
@loaded_content[:desired_contents]
|
196
179
|
end
|
197
180
|
|
198
181
|
def purge_exclusions
|
199
|
-
|
200
|
-
|
201
|
-
if environment && environment.respond_to?(:desired_contents)
|
202
|
-
exclusions += environment.desired_contents
|
203
|
-
end
|
182
|
+
self.load
|
204
183
|
|
205
|
-
|
184
|
+
@loaded_content[:purge_exclusions]
|
206
185
|
end
|
207
186
|
|
208
187
|
def accept(visitor)
|
@@ -225,65 +204,10 @@ class Puppetfile
|
|
225
204
|
|
226
205
|
private
|
227
206
|
|
228
|
-
def puppetfile_contents
|
229
|
-
File.read(@puppetfile_path)
|
230
|
-
end
|
231
|
-
|
232
|
-
def resolve_install_path(path)
|
233
|
-
pn = Pathname.new(path)
|
234
|
-
|
235
|
-
unless pn.absolute?
|
236
|
-
pn = Pathname.new(File.join(basedir, path))
|
237
|
-
end
|
238
|
-
|
239
|
-
# .cleanpath is as good as we can do without touching the filesystem.
|
240
|
-
# The .realpath methods will also choke if some of the intermediate
|
241
|
-
# paths are missing, even though we will create them later as needed.
|
242
|
-
pn.cleanpath.to_s
|
243
|
-
end
|
244
|
-
|
245
|
-
def validate_install_path(path, modname)
|
246
|
-
unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
|
247
|
-
raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
|
248
|
-
end
|
249
|
-
|
250
|
-
true
|
251
|
-
end
|
252
|
-
|
253
207
|
def real_basedir
|
254
208
|
Pathname.new(basedir).cleanpath.to_s
|
255
209
|
end
|
256
210
|
|
257
|
-
|
258
|
-
# A barebones implementation of the Puppetfile DSL
|
259
|
-
#
|
260
|
-
# @api private
|
261
|
-
|
262
|
-
def initialize(librarian)
|
263
|
-
@librarian = librarian
|
264
|
-
end
|
265
|
-
|
266
|
-
def mod(name, args = nil)
|
267
|
-
if args.is_a?(Hash)
|
268
|
-
opts = args
|
269
|
-
else
|
270
|
-
opts = { version: args }
|
271
|
-
end
|
272
|
-
|
273
|
-
@librarian.add_module(name, opts)
|
274
|
-
end
|
275
|
-
|
276
|
-
def forge(location)
|
277
|
-
@librarian.set_forge(location)
|
278
|
-
end
|
279
|
-
|
280
|
-
def moduledir(location)
|
281
|
-
@librarian.set_moduledir(location)
|
282
|
-
end
|
283
|
-
|
284
|
-
def method_missing(method, *args)
|
285
|
-
raise NoMethodError, _("unrecognized declaration '%{method}'") % {method: method}
|
286
|
-
end
|
287
|
-
end
|
211
|
+
DSL = R10K::ModuleLoader::Puppetfile::DSL
|
288
212
|
end
|
289
213
|
end
|
data/lib/r10k/settings.rb
CHANGED
@@ -92,6 +92,9 @@ module R10K
|
|
92
92
|
URIDefinition.new(:baseurl, {
|
93
93
|
:desc => "The URL to the Puppet Forge to use for downloading modules."
|
94
94
|
}),
|
95
|
+
Definition.new(:authorization_token, {
|
96
|
+
:desc => "The token for Puppet Forge authorization. Leave blank for unauthorized or license-based connections."
|
97
|
+
})
|
95
98
|
])
|
96
99
|
end
|
97
100
|
|
data/lib/r10k/source/base.rb
CHANGED
@@ -63,6 +63,16 @@ class R10K::Source::Base
|
|
63
63
|
|
64
64
|
end
|
65
65
|
|
66
|
+
# Perform actions to reload environments after the `preload!`. Similar
|
67
|
+
# to preload!, and likely to include network queries and rerunning
|
68
|
+
# environment generation.
|
69
|
+
#
|
70
|
+
# @api public
|
71
|
+
# @abstract
|
72
|
+
# @return [void]
|
73
|
+
def reload!
|
74
|
+
end
|
75
|
+
|
66
76
|
# Enumerate the environments associated with this SVN source.
|
67
77
|
#
|
68
78
|
# @api public
|
data/lib/r10k/source/git.rb
CHANGED
data/lib/r10k/source/svn.rb
CHANGED
@@ -65,6 +65,10 @@ class R10K::Source::SVN < R10K::Source::Base
|
|
65
65
|
@ignore_branch_prefixes = options[:ignore_branch_prefixes]
|
66
66
|
end
|
67
67
|
|
68
|
+
def reload!
|
69
|
+
@environments = generate_environments()
|
70
|
+
end
|
71
|
+
|
68
72
|
# Enumerate the environments associated with this SVN source.
|
69
73
|
#
|
70
74
|
# @return [Array<R10K::Environment::SVN>] An array of environments created
|
data/lib/r10k/util/purgeable.rb
CHANGED
@@ -9,6 +9,12 @@ module R10K
|
|
9
9
|
# {#desired_contents}
|
10
10
|
module Purgeable
|
11
11
|
|
12
|
+
HIDDEN_FILE = /\.[^.]+/
|
13
|
+
|
14
|
+
FN_MATCH_OPTS = File::FNM_PATHNAME | File::FNM_DOTMATCH
|
15
|
+
|
16
|
+
# @deprecated
|
17
|
+
#
|
12
18
|
# @!method logger
|
13
19
|
# @abstract Including classes must provide a logger method
|
14
20
|
# @return [Log4r::Logger]
|
@@ -38,23 +44,79 @@ module R10K
|
|
38
44
|
end
|
39
45
|
end
|
40
46
|
|
47
|
+
# @deprecated Unused helper function
|
48
|
+
#
|
41
49
|
# @return [Array<String>] Directory contents that are expected but not present
|
42
50
|
def pending_contents(recurse)
|
43
51
|
desired_contents - current_contents(recurse)
|
44
52
|
end
|
45
53
|
|
54
|
+
def matches?(test, path)
|
55
|
+
if test == path
|
56
|
+
true
|
57
|
+
elsif File.fnmatch?(test, path, FN_MATCH_OPTS)
|
58
|
+
true
|
59
|
+
else
|
60
|
+
false
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# A method to collect potentially purgeable content without searching into
|
65
|
+
# ignored directories when recursively searching.
|
66
|
+
#
|
67
|
+
# @param dir [String, Pathname] The directory to search for purgeable content
|
68
|
+
# @param exclusion_gobs [Array<String>] A list of file paths or File globs
|
69
|
+
# to exclude from recursion (These are generated by the classes that
|
70
|
+
# mix this module into them and are typically programatically generated)
|
71
|
+
# @param allowed_gobs [Array<String>] A list of file paths or File globs to exclude
|
72
|
+
# from recursion (These are passed in by the caller of purge! and typically
|
73
|
+
# are user supplied configuration values)
|
74
|
+
# @param desireds_not_to_recurse_into [Array<String>] A list of file paths not to
|
75
|
+
# recurse into. These are programatically generated, these exist to maintain
|
76
|
+
# backwards compatibility with previous implementations that used File.globs
|
77
|
+
# for "recursion", ie "**/{*,.[^.]*}" which would not recurse into dot directories.
|
78
|
+
# @param recurse [Boolean] Whether or not to recurse into child directories that do
|
79
|
+
# not match other filters.
|
80
|
+
#
|
81
|
+
# @return [Array<String>] Contents which may be purged.
|
82
|
+
def potentially_purgeable(dir, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
|
83
|
+
children = Pathname.new(dir).children.reject do |path|
|
84
|
+
path = path.to_s
|
85
|
+
|
86
|
+
if exclusion_match = exclusion_globs.find { |exclusion| matches?(exclusion, path) }
|
87
|
+
logger.debug2 _("Not purging %{path} due to internal exclusion match: %{exclusion_match}") % {path: path, exclusion_match: exclusion_match}
|
88
|
+
elsif allowlist_match = allowed_globs.find { |allowed| matches?(allowed, path) }
|
89
|
+
logger.debug _("Not purging %{path} due to whitelist match: %{allowlist_match}") % {path: path, allowlist_match: allowlist_match}
|
90
|
+
else
|
91
|
+
desired_match = desireds_not_to_recurse_into.grep(path).first
|
92
|
+
end
|
93
|
+
|
94
|
+
!!exclusion_match || !!allowlist_match || !!desired_match
|
95
|
+
end
|
96
|
+
|
97
|
+
children.flat_map do |child|
|
98
|
+
if File.directory?(child) && recurse
|
99
|
+
potentially_purgeable(child, exclusion_globs, allowed_globs, desireds_not_to_recurse_into, recurse)
|
100
|
+
else
|
101
|
+
child.to_s
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
46
106
|
# @return [Array<String>] Directory contents that are present but not expected
|
47
107
|
def stale_contents(recurse, exclusions, whitelist)
|
48
|
-
|
108
|
+
dirs = self.managed_directories
|
109
|
+
desireds = self.desired_contents
|
110
|
+
hidden_desireds, regular_desireds = desireds.partition do |desired|
|
111
|
+
HIDDEN_FILE.match(File.basename(desired))
|
112
|
+
end
|
49
113
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
elsif whitelist_match = whitelist.find { |wl_item| (wl_item == item) || File.fnmatch?(wl_item, item, fn_match_opts) }
|
54
|
-
logger.debug _("Not purging %{item} due to whitelist match: %{whitelist_match}") % {item: item, whitelist_match: whitelist_match}
|
55
|
-
end
|
114
|
+
initial_purgelist = dirs.flat_map do |dir|
|
115
|
+
potentially_purgeable(dir, exclusions, whitelist, hidden_desireds, recurse)
|
116
|
+
end
|
56
117
|
|
57
|
-
|
118
|
+
initial_purgelist.reject do |path|
|
119
|
+
regular_desireds.any? { |desired| matches?(desired, path) }
|
58
120
|
end
|
59
121
|
end
|
60
122
|
|