r10k 3.5.0 → 3.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/.github/pull_request_template.md +4 -1
  3. data/.github/workflows/docker.yml +25 -1
  4. data/.github/workflows/rspec_tests.yml +81 -0
  5. data/.travis.yml +14 -11
  6. data/CHANGELOG.mkd +42 -6
  7. data/CODEOWNERS +1 -1
  8. data/Gemfile +1 -1
  9. data/README.mkd +13 -4
  10. data/azure-pipelines.yml +2 -1
  11. data/doc/dynamic-environments/configuration.mkd +60 -3
  12. data/doc/dynamic-environments/usage.mkd +5 -4
  13. data/doc/faq.mkd +6 -1
  14. data/doc/puppetfile.mkd +2 -0
  15. data/docker/Makefile +16 -2
  16. data/docker/r10k/Dockerfile +17 -6
  17. data/docker/r10k/release.Dockerfile +23 -4
  18. data/integration/tests/git_source/git_source_repeated_remote.rb +68 -0
  19. data/integration/tests/user_scenario/complex_workflow/multi_env_add_change_remove.rb +1 -1
  20. data/integration/tests/user_scenario/complex_workflow/multi_env_remove_re-add.rb +1 -1
  21. data/integration/tests/user_scenario/complex_workflow/multi_env_unamanaged.rb +1 -1
  22. data/lib/r10k/action/deploy/environment.rb +3 -0
  23. data/lib/r10k/action/deploy/module.rb +4 -1
  24. data/lib/r10k/action/runner.rb +34 -0
  25. data/lib/r10k/cli/deploy.rb +9 -4
  26. data/lib/r10k/cli/puppetfile.rb +5 -5
  27. data/lib/r10k/environment/base.rb +8 -1
  28. data/lib/r10k/environment/with_modules.rb +27 -19
  29. data/lib/r10k/forge/module_release.rb +2 -2
  30. data/lib/r10k/git.rb +1 -0
  31. data/lib/r10k/git/cache.rb +12 -4
  32. data/lib/r10k/git/rugged/credentials.rb +32 -2
  33. data/lib/r10k/git/stateful_repository.rb +4 -0
  34. data/lib/r10k/initializers.rb +2 -0
  35. data/lib/r10k/module/base.rb +8 -0
  36. data/lib/r10k/module/forge.rb +1 -1
  37. data/lib/r10k/module/git.rb +20 -3
  38. data/lib/r10k/puppetfile.rb +30 -12
  39. data/lib/r10k/settings.rb +24 -2
  40. data/lib/r10k/source/git.rb +22 -2
  41. data/lib/r10k/version.rb +1 -1
  42. data/locales/r10k.pot +60 -36
  43. data/spec/fixtures/unit/action/r10k_creds.yaml +9 -0
  44. data/spec/shared-examples/subprocess-runner.rb +11 -5
  45. data/spec/unit/action/deploy/environment_spec.rb +43 -2
  46. data/spec/unit/action/deploy/module_spec.rb +40 -1
  47. data/spec/unit/action/puppetfile/install_spec.rb +1 -0
  48. data/spec/unit/action/runner_spec.rb +48 -1
  49. data/spec/unit/environment/git_spec.rb +3 -2
  50. data/spec/unit/environment/with_modules_spec.rb +74 -0
  51. data/spec/unit/forge/module_release_spec.rb +14 -10
  52. data/spec/unit/git/cache_spec.rb +10 -0
  53. data/spec/unit/git/rugged/credentials_spec.rb +69 -2
  54. data/spec/unit/git_spec.rb +3 -3
  55. data/spec/unit/module/git_spec.rb +55 -0
  56. data/spec/unit/puppetfile_spec.rb +61 -7
  57. data/spec/unit/settings_spec.rb +12 -0
  58. data/spec/unit/source/git_spec.rb +49 -1
  59. metadata +6 -2
@@ -103,6 +103,13 @@ class R10K::Environment::Base
103
103
  @puppetfile.modules
104
104
  end
105
105
 
106
+ # @return [Array<R10K::Module::Base>] Whether or not the given module
107
+ # conflicts with any modules already defined in the r10k environment
108
+ # object.
109
+ def module_conflicts?(mod)
110
+ false
111
+ end
112
+
106
113
  def accept(visitor)
107
114
  visitor.visit(:environment, self) do
108
115
  puppetfile.accept(visitor)
@@ -137,7 +144,7 @@ class R10K::Environment::Base
137
144
  end
138
145
 
139
146
  def generate_types!
140
- argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir]
147
+ argv = [R10K::Settings.puppet_path, 'generate', 'types', '--environment', dirname, '--environmentpath', basedir, '--config', R10K::Settings.puppet_conf]
141
148
  subproc = R10K::Util::Subprocess.new(argv)
142
149
  subproc.raise_on_fail = true
143
150
  subproc.logger = logger
@@ -46,10 +46,33 @@ class R10K::Environment::WithModules < R10K::Environment::Base
46
46
  # - The r10k environment object
47
47
  # - A Puppetfile in the environment's content
48
48
  def modules
49
- return @modules if @puppetfile.nil?
49
+ return @modules if puppetfile.nil?
50
50
 
51
- @puppetfile.load unless @puppetfile.loaded?
52
- @modules + @puppetfile.modules
51
+ puppetfile.load unless puppetfile.loaded?
52
+ @modules + puppetfile.modules
53
+ end
54
+
55
+ def module_conflicts?(mod_b)
56
+ conflict = @modules.any? { |mod_a| mod_a.name == mod_b.name }
57
+ return false unless conflict
58
+
59
+ msg_vars = {src: mod_b.origin, name: mod_b.name}
60
+ msg_error = _('Environment and %{src} both define the "%{name}" module' % msg_vars)
61
+ msg_continue = _("#{msg_error}. The %{src} definition will be ignored" % msg_vars)
62
+
63
+ case conflict_opt = @options[:module_conflicts]
64
+ when 'override_and_warn', nil
65
+ logger.warn msg_continue
66
+ when 'override'
67
+ logger.debug msg_continue
68
+ when 'error'
69
+ raise R10K::Error, msg_error
70
+ else
71
+ raise R10K::Error, _('Unexpected value for `module_conflicts` setting in %{env} ' \
72
+ 'environment: %{val}' % {env: self.name, val: conflict_opt})
73
+ end
74
+
75
+ true
53
76
  end
54
77
 
55
78
  def accept(visitor)
@@ -59,7 +82,6 @@ class R10K::Environment::WithModules < R10K::Environment::Base
59
82
  end
60
83
 
61
84
  puppetfile.accept(visitor)
62
- validate_no_module_conflicts
63
85
  end
64
86
  end
65
87
 
@@ -88,26 +110,12 @@ class R10K::Environment::WithModules < R10K::Environment::Base
88
110
  @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
89
111
 
90
112
  mod = R10K::Module.new(name, install_path, args, self.name)
91
- mod.origin = 'Environment'
113
+ mod.origin = :environment
92
114
 
93
115
  @managed_content[install_path] << mod.name
94
116
  @modules << mod
95
117
  end
96
118
 
97
- def validate_no_module_conflicts
98
- @puppetfile.load unless @puppetfile.loaded?
99
- conflicts = (@modules + @puppetfile.modules)
100
- .group_by { |mod| mod.name }
101
- .select { |_, v| v.size > 1 }
102
- .map(&:first)
103
- unless conflicts.empty?
104
- msg = _('Puppetfile cannot contain module names defined by environment %{name}') % {name: self.name}
105
- msg += ' '
106
- msg += _("Remove the conflicting definitions of the following modules: %{conflicts}" % { conflicts: conflicts.join(' ') })
107
- raise R10K::Error.new(msg)
108
- end
109
- end
110
-
111
119
  include R10K::Util::Purgeable
112
120
 
113
121
  # Returns an array of the full paths that can be purged.
@@ -212,14 +212,14 @@ module R10K
212
212
 
213
213
  # Remove the temporary directory used for unpacking the module.
214
214
  def cleanup_unpack_path
215
- if unpack_path.exist?
215
+ if unpack_path.parent.exist?
216
216
  unpack_path.parent.rmtree
217
217
  end
218
218
  end
219
219
 
220
220
  # Remove the downloaded module release.
221
221
  def cleanup_download_path
222
- if download_path.exist?
222
+ if download_path.parent.exist?
223
223
  download_path.parent.rmtree
224
224
  end
225
225
  end
data/lib/r10k/git.rb CHANGED
@@ -134,6 +134,7 @@ module R10K
134
134
  extend R10K::Settings::Mixin::ClassMethods
135
135
 
136
136
  def_setting_attr :private_key
137
+ def_setting_attr :oauth_token
137
138
  def_setting_attr :proxy
138
139
  def_setting_attr :username
139
140
  def_setting_attr :repositories, {}
@@ -16,7 +16,17 @@ class R10K::Git::Cache
16
16
 
17
17
  include R10K::Settings::Mixin
18
18
 
19
- def_setting_attr :cache_root, File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
19
+ #@api private
20
+ def self.determine_cache_root
21
+ if R10K::Util::Platform.windows?
22
+ File.join(ENV['LOCALAPPDATA'], 'r10k', 'git')
23
+ else
24
+ File.expand_path(ENV['HOME'] ? '~/.r10k/git': '/root/.r10k/git')
25
+ end
26
+ end
27
+ private_class_method :determine_cache_root
28
+
29
+ def_setting_attr :cache_root, determine_cache_root
20
30
 
21
31
  @instance_cache = R10K::InstanceCache.new(self)
22
32
 
@@ -99,10 +109,8 @@ class R10K::Git::Cache
99
109
 
100
110
  alias cached? exist?
101
111
 
102
- private
103
-
104
112
  # Reformat the remote name into something that can be used as a directory
105
113
  def sanitized_dirname
106
- @remote.gsub(/[^@\w\.-]/, '-')
114
+ @sanitized_dirname ||= @remote.gsub(/[^@\w\.-]/, '-')
107
115
  end
108
116
  end
@@ -61,11 +61,41 @@ class R10K::Git::Rugged::Credentials
61
61
  end
62
62
 
63
63
  def get_plaintext_credentials(url, username_from_url)
64
- user = get_git_username(url, username_from_url)
65
- password = URI.parse(url).password || ''
64
+ per_repo_oauth_token = nil
65
+ if per_repo_settings = R10K::Git.get_repo_settings(url)
66
+ per_repo_oauth_token = per_repo_settings[:oauth_token]
67
+ end
68
+
69
+ if token_path = per_repo_oauth_token || R10K::Git.settings[:oauth_token]
70
+ if token_path == '-'
71
+ token = $stdin.read.strip
72
+ logger.debug2 _("Using OAuth token from stdin for URL %{url}") % { url: url }
73
+ elsif File.readable?(token_path)
74
+ token = File.read(token_path).strip
75
+ logger.debug2 _("Using OAuth token from %{token_path} for URL %{url}") % { token_path: token_path, url: url }
76
+ else
77
+ raise R10K::Git::GitError, _("%{path} is missing or unreadable, cannot load OAuth token") % { path: token_path }
78
+ end
79
+
80
+ unless valid_token?(token)
81
+ raise R10K::Git::GitError, _("Supplied OAuth token contains invalid characters.")
82
+ end
83
+
84
+ user = 'x-oauth-token'
85
+ password = token
86
+ else
87
+ user = get_git_username(url, username_from_url)
88
+ password = URI.parse(url).password || ''
89
+ end
66
90
  Rugged::Credentials::UserPassword.new(username: user, password: password)
67
91
  end
68
92
 
93
+ # This regex is the only real requirement for OAuth token format,
94
+ # per https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
95
+ def valid_token?(token)
96
+ return token =~ /^[\w\-\.~\+\/]+$/
97
+ end
98
+
69
99
  def get_default_credentials(url, username_from_url)
70
100
  Rugged::Credentials::Default.new
71
101
  end
@@ -12,6 +12,10 @@ class R10K::Git::StatefulRepository
12
12
  # @api private
13
13
  attr_reader :repo
14
14
 
15
+ # @!attribute [r] cache
16
+ # @api private
17
+ attr_reader :cache
18
+
15
19
  extend Forwardable
16
20
  def_delegators :@repo, :head, :tracked_paths
17
21
 
@@ -44,6 +44,7 @@ module R10K
44
44
  class DeployInitializer < BaseInitializer
45
45
  def call
46
46
  with_setting(:puppet_path) { |value| R10K::Settings.puppet_path = value }
47
+ with_setting(:puppet_conf) { |value| R10K::Settings.puppet_conf = value }
47
48
  end
48
49
  end
49
50
 
@@ -54,6 +55,7 @@ module R10K
54
55
  with_setting(:private_key) { |value| R10K::Git.settings[:private_key] = value }
55
56
  with_setting(:proxy) { |value| R10K::Git.settings[:proxy] = value }
56
57
  with_setting(:repositories) { |value| R10K::Git.settings[:repositories] = value }
58
+ with_setting(:oauth_token) { |value| R10K::Git.settings[:oauth_token] = value }
57
59
  end
58
60
  end
59
61
 
@@ -99,6 +99,14 @@ class R10K::Module::Base
99
99
  raise NotImplementedError
100
100
  end
101
101
 
102
+ # Return the module's cachedir. Subclasses that implement a cache
103
+ # will override this to return a real directory location.
104
+ #
105
+ # @return [String, :none]
106
+ def cachedir
107
+ :none
108
+ end
109
+
102
110
  private
103
111
 
104
112
  def parse_title(title)
@@ -171,7 +171,7 @@ class R10K::Module::Forge < R10K::Module::Base
171
171
  if (match = title.match(/\A(\w+)[-\/](\w+)\Z/))
172
172
  [match[1], match[2]]
173
173
  else
174
- raise ArgumentError, _("Forge module names must match 'owner/modulename'")
174
+ raise ArgumentError, _("Forge module names must match 'owner/modulename', instead got #{title}")
175
175
  end
176
176
  end
177
177
  end
@@ -28,6 +28,11 @@ class R10K::Module::Git < R10K::Module::Base
28
28
  # @return [String]
29
29
  attr_reader :default_ref
30
30
 
31
+ # @!attribute [r] default_override_ref
32
+ # @api private
33
+ # @return [String]
34
+ attr_reader :default_override_ref
35
+
31
36
  def initialize(title, dirname, args, environment=nil)
32
37
  super
33
38
 
@@ -37,7 +42,7 @@ class R10K::Module::Git < R10K::Module::Base
37
42
  end
38
43
 
39
44
  def version
40
- validate_ref(@desired_ref, @default_ref)
45
+ validate_ref(@desired_ref, @default_ref, @default_override_ref)
41
46
  end
42
47
 
43
48
  def properties
@@ -57,11 +62,17 @@ class R10K::Module::Git < R10K::Module::Base
57
62
  @repo.status(version)
58
63
  end
59
64
 
65
+ def cachedir
66
+ @repo.cache.sanitized_dirname
67
+ end
68
+
60
69
  private
61
70
 
62
- def validate_ref(desired, default)
71
+ def validate_ref(desired, default, default_override)
63
72
  if desired && desired != :control_branch && @repo.resolve(desired)
64
73
  return desired
74
+ elsif default_override && @repo.resolve(default_override)
75
+ return default_override
65
76
  elsif default && @repo.resolve(default)
66
77
  return default
67
78
  else
@@ -77,6 +88,11 @@ class R10K::Module::Git < R10K::Module::Base
77
88
  msg << "Could not determine desired ref"
78
89
  end
79
90
 
91
+ if default_override
92
+ msg << "or resolve the default branch override '%{default_override}',"
93
+ vars[:default_override] = default_override
94
+ end
95
+
80
96
  if default
81
97
  msg << "or resolve default ref '%{default}'"
82
98
  vars[:default] = default
@@ -90,7 +106,7 @@ class R10K::Module::Git < R10K::Module::Base
90
106
 
91
107
  def parse_options(options)
92
108
  ref_opts = [:branch, :tag, :commit, :ref]
93
- known_opts = [:git, :default_branch] + ref_opts
109
+ known_opts = [:git, :default_branch, :default_branch_override] + ref_opts
94
110
 
95
111
  unhandled = options.keys - known_opts
96
112
  unless unhandled.empty?
@@ -101,6 +117,7 @@ class R10K::Module::Git < R10K::Module::Base
101
117
 
102
118
  @desired_ref = ref_opts.find { |key| break options[key] if options.has_key?(key) } || 'master'
103
119
  @default_ref = options[:default_branch]
120
+ @default_override_ref = options[:default_branch_override]
104
121
 
105
122
  if @desired_ref == :control_branch && @environment && @environment.respond_to?(:ref)
106
123
  @desired_ref = @environment.ref
@@ -10,7 +10,7 @@ class Puppetfile
10
10
 
11
11
  include R10K::Settings::Mixin
12
12
 
13
- def_setting_attr :pool_size, 1
13
+ def_setting_attr :pool_size, 4
14
14
 
15
15
  include R10K::Logging
16
16
 
@@ -77,7 +77,7 @@ class Puppetfile
77
77
 
78
78
  dsl = R10K::Puppetfile::DSL.new(self)
79
79
  dsl.instance_eval(puppetfile_contents, @puppetfile_path)
80
-
80
+
81
81
  validate_no_duplicate_names(@modules)
82
82
  @loaded = true
83
83
  rescue SyntaxError, LoadError, ArgumentError, NameError => e
@@ -127,16 +127,23 @@ class Puppetfile
127
127
  end
128
128
 
129
129
  if args.is_a?(Hash) && @default_branch_override != nil
130
- args[:default_branch] = @default_branch_override
130
+ args[:default_branch_override] = @default_branch_override
131
131
  end
132
132
 
133
- # Keep track of all the content this Puppetfile is managing to enable purging.
134
- @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
135
-
136
133
  mod = R10K::Module.new(name, install_path, args, @environment)
137
- mod.origin = 'Puppetfile'
134
+ mod.origin = :puppetfile
138
135
 
136
+ # Do not load modules if they would conflict with the attached
137
+ # environment
138
+ if environment && environment.module_conflicts?(mod)
139
+ mod = nil
140
+ return @modules
141
+ end
142
+
143
+ # Keep track of all the content this Puppetfile is managing to enable purging.
144
+ @managed_content[install_path] = Array.new unless @managed_content.has_key?(install_path)
139
145
  @managed_content[install_path] << mod.name
146
+
140
147
  @modules << mod
141
148
  end
142
149
 
@@ -145,7 +152,9 @@ class Puppetfile
145
152
  def managed_directories
146
153
  self.load unless @loaded
147
154
 
148
- @managed_content.keys
155
+ dirs = @managed_content.keys
156
+ dirs.delete(real_basedir)
157
+ dirs
149
158
  end
150
159
 
151
160
  # Returns an array of the full paths to all the content being managed.
@@ -212,15 +221,22 @@ class Puppetfile
212
221
  def modules_queue(visitor)
213
222
  Queue.new.tap do |queue|
214
223
  visitor.visit(:puppetfile, self) do
215
- modules.each { |mod| queue << mod }
224
+ modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
225
+ modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
226
+
227
+ modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
228
+ modules_by_cachedir.values.each {|mods| queue << mods }
216
229
  end
217
230
  end
218
231
  end
232
+ public :modules_queue
219
233
 
220
234
  def visitor_thread(visitor, mods_queue)
221
235
  Thread.new do
222
236
  begin
223
- while mod = mods_queue.pop(true) do mod.accept(visitor) end
237
+ while mods = mods_queue.pop(true) do
238
+ mods.each {|mod| mod.accept(visitor) }
239
+ end
224
240
  rescue ThreadError => e
225
241
  logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
226
242
  Thread.exit
@@ -248,8 +264,6 @@ class Puppetfile
248
264
  end
249
265
 
250
266
  def validate_install_path(path, modname)
251
- real_basedir = Pathname.new(basedir).cleanpath.to_s
252
-
253
267
  unless /^#{Regexp.escape(real_basedir)}.*/ =~ path
254
268
  raise R10K::Error.new("Puppetfile cannot manage content '#{modname}' outside of containing environment: #{path} is not within #{real_basedir}")
255
269
  end
@@ -257,6 +271,10 @@ class Puppetfile
257
271
  true
258
272
  end
259
273
 
274
+ def real_basedir
275
+ Pathname.new(basedir).cleanpath.to_s
276
+ end
277
+
260
278
  class DSL
261
279
  # A barebones implementation of the Puppetfile DSL
262
280
  #
data/lib/r10k/settings.rb CHANGED
@@ -12,6 +12,8 @@ module R10K
12
12
  class << self
13
13
  # Path to puppet executable
14
14
  attr_accessor :puppet_path
15
+ # Path to puppet.conf
16
+ attr_accessor :puppet_conf
15
17
  end
16
18
 
17
19
  def self.git_settings
@@ -35,6 +37,11 @@ module R10K
35
37
  Only used by the 'rugged' Git provider.",
36
38
  }),
37
39
 
40
+ Definition.new(:oauth_token, {
41
+ :desc => "The path to a token file for Git OAuth remotes.
42
+ Only used by the 'rugged' Git provider."
43
+ }),
44
+
38
45
  URIDefinition.new(:proxy, {
39
46
  :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
40
47
  :default => :inherit,
@@ -52,11 +59,17 @@ module R10K
52
59
  :default => :inherit,
53
60
  }),
54
61
 
62
+ Definition.new(:oauth_token, {
63
+ :desc => "The path to a token file for Git OAuth remotes.
64
+ Only used by the 'rugged' Git provider.",
65
+ :default => :inherit
66
+ }),
67
+
55
68
  URIDefinition.new(:proxy, {
56
69
  :desc => "An optional proxy server to use when interacting with Git sources via HTTP(S).",
57
70
  :default => :inherit,
58
71
  }),
59
-
72
+
60
73
  Definition.new(:ignore_branch_prefixes, {
61
74
  :desc => "Array of strings used to prefix branch names that will not be deployed as environments.",
62
75
  }),
@@ -131,6 +144,15 @@ module R10K
131
144
  end
132
145
  end
133
146
  }),
147
+ Definition.new(:puppet_conf, {
148
+ :desc => "Path to puppet.conf. Defaults to /etc/puppetlabs/puppet/puppet.conf.",
149
+ :default => '/etc/puppetlabs/puppet/puppet.conf',
150
+ :validate => lambda do |value|
151
+ unless File.readable? value
152
+ raise ArgumentError, "The specified puppet.conf #{value} is not readable"
153
+ end
154
+ end
155
+ }),
134
156
  ])
135
157
  end
136
158
 
@@ -160,7 +182,7 @@ module R10K
160
182
 
161
183
  Definition.new(:pool_size, {
162
184
  :desc => "The amount of threads used to concurrently install modules. The default value is 1: install one module at a time.",
163
- :default => 1,
185
+ :default => 4,
164
186
  :validate => lambda do |value|
165
187
  if !value.is_a?(Integer)
166
188
  raise ArgumentError, "The pool_size setting should be an integer, not a #{value.class}"