r10k 3.7.0 → 3.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.github/pull_request_template.md +1 -1
  3. data/.github/workflows/docker.yml +4 -1
  4. data/.github/workflows/release.yml +3 -2
  5. data/.github/workflows/rspec_tests.yml +1 -1
  6. data/.github/workflows/stale.yml +19 -0
  7. data/.travis.yml +8 -1
  8. data/CHANGELOG.mkd +32 -0
  9. data/CODEOWNERS +2 -2
  10. data/doc/common-patterns.mkd +1 -0
  11. data/doc/dynamic-environments/configuration.mkd +114 -42
  12. data/doc/dynamic-environments/usage.mkd +12 -11
  13. data/doc/puppetfile.mkd +23 -3
  14. data/docker/Gemfile +1 -1
  15. data/docker/Makefile +4 -3
  16. data/docker/docker-compose.yml +18 -0
  17. data/docker/r10k/Dockerfile +1 -1
  18. data/docker/r10k/docker-entrypoint.sh +0 -1
  19. data/docker/r10k/release.Dockerfile +1 -1
  20. data/docker/spec/dockerfile_spec.rb +26 -32
  21. data/integration/tests/git_source/git_source_repeated_remote.rb +2 -2
  22. data/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module.rb +2 -1
  23. data/integration/tests/user_scenario/basic_workflow/multi_env_custom_forge_git_module_static.rb +2 -1
  24. data/integration/tests/user_scenario/basic_workflow/multi_source_custom_forge_git_module.rb +1 -1
  25. data/integration/tests/user_scenario/basic_workflow/single_env_custom_forge_git_module.rb +2 -1
  26. data/lib/r10k/action/base.rb +10 -0
  27. data/lib/r10k/action/deploy/display.rb +49 -10
  28. data/lib/r10k/action/deploy/environment.rb +101 -51
  29. data/lib/r10k/action/deploy/module.rb +54 -30
  30. data/lib/r10k/action/puppetfile/check.rb +3 -1
  31. data/lib/r10k/action/puppetfile/install.rb +20 -23
  32. data/lib/r10k/action/puppetfile/purge.rb +8 -2
  33. data/lib/r10k/action/runner.rb +33 -0
  34. data/lib/r10k/cli/deploy.rb +13 -7
  35. data/lib/r10k/cli/puppetfile.rb +5 -5
  36. data/lib/r10k/content_synchronizer.rb +83 -0
  37. data/lib/r10k/deployment.rb +1 -1
  38. data/lib/r10k/environment/base.rb +29 -2
  39. data/lib/r10k/environment/git.rb +17 -5
  40. data/lib/r10k/environment/name.rb +22 -4
  41. data/lib/r10k/environment/svn.rb +11 -4
  42. data/lib/r10k/environment/with_modules.rb +46 -30
  43. data/lib/r10k/git.rb +1 -0
  44. data/lib/r10k/git/rugged/credentials.rb +39 -2
  45. data/lib/r10k/initializers.rb +1 -0
  46. data/lib/r10k/module.rb +1 -1
  47. data/lib/r10k/module/base.rb +17 -1
  48. data/lib/r10k/module/forge.rb +29 -11
  49. data/lib/r10k/module/git.rb +50 -27
  50. data/lib/r10k/module/local.rb +2 -1
  51. data/lib/r10k/module/svn.rb +24 -18
  52. data/lib/r10k/puppetfile.rb +66 -83
  53. data/lib/r10k/settings.rb +18 -2
  54. data/lib/r10k/source/base.rb +9 -0
  55. data/lib/r10k/source/git.rb +18 -7
  56. data/lib/r10k/source/hash.rb +5 -5
  57. data/lib/r10k/source/svn.rb +5 -3
  58. data/lib/r10k/util/cleaner.rb +21 -0
  59. data/lib/r10k/util/setopts.rb +33 -12
  60. data/lib/r10k/version.rb +1 -1
  61. data/locales/r10k.pot +98 -82
  62. data/r10k.gemspec +1 -1
  63. data/spec/fixtures/unit/action/r10k_creds.yaml +9 -0
  64. data/spec/r10k-mocks/mock_source.rb +1 -1
  65. data/spec/shared-examples/puppetfile-action.rb +7 -7
  66. data/spec/unit/action/deploy/display_spec.rb +35 -5
  67. data/spec/unit/action/deploy/environment_spec.rb +199 -38
  68. data/spec/unit/action/deploy/module_spec.rb +162 -28
  69. data/spec/unit/action/puppetfile/check_spec.rb +2 -2
  70. data/spec/unit/action/puppetfile/install_spec.rb +31 -10
  71. data/spec/unit/action/puppetfile/purge_spec.rb +25 -5
  72. data/spec/unit/action/runner_spec.rb +48 -1
  73. data/spec/unit/environment/git_spec.rb +19 -2
  74. data/spec/unit/environment/name_spec.rb +28 -0
  75. data/spec/unit/environment/svn_spec.rb +12 -0
  76. data/spec/unit/environment/with_modules_spec.rb +74 -0
  77. data/spec/unit/git/rugged/credentials_spec.rb +78 -1
  78. data/spec/unit/module/forge_spec.rb +21 -13
  79. data/spec/unit/module/git_spec.rb +63 -8
  80. data/spec/unit/module_spec.rb +77 -10
  81. data/spec/unit/puppetfile_spec.rb +63 -60
  82. data/spec/unit/util/purgeable_spec.rb +2 -8
  83. data/spec/unit/util/setopts_spec.rb +25 -1
  84. metadata +11 -12
  85. data/azure-pipelines.yml +0 -87
@@ -10,25 +10,56 @@ module R10K
10
10
 
11
11
  include R10K::Action::Deploy::DeployHelpers
12
12
 
13
+ # Deprecated
13
14
  attr_reader :force
14
15
 
15
- def initialize(opts, argv, settings = nil)
16
- settings ||= {}
17
-
16
+ attr_reader :settings
17
+
18
+ # @param opts [Hash] A hash of options defined in #allowed_initialized_opts
19
+ # and managed by the SetOps mixin within the Action::Base class.
20
+ # Corresponds to the CLI flags and options.
21
+ # @param argv [Enumerable] Typically CRI::ArgumentList or Array. A list-like
22
+ # collection of the remaining arguments to the CLI invocation (after
23
+ # removing flags and options).
24
+ # @param settings [Hash] A hash of configuration loaded from the relevant
25
+ # config (r10k.yaml).
26
+ #
27
+ # @note All arguments will be required in the next major version
28
+ def initialize(opts, argv, settings = {})
18
29
  super
19
30
 
20
- # @force here is used to make it easier to reason about
21
- @force = !@no_force
31
+ requested_env = @opts[:environment] ? [@opts[:environment].gsub(/\W/, '_')] : []
32
+
33
+ @settings = @settings.merge({
34
+ overrides: {
35
+ environments: {
36
+ requested_environments: requested_env,
37
+ generate_types: @generate_types
38
+ },
39
+ modules: {
40
+ requested_modules: @argv.map.to_a,
41
+ # force here is used to make it easier to reason about
42
+ force: !@no_force
43
+ },
44
+ purging: {},
45
+ output: {}
46
+ }
47
+ })
22
48
  end
23
49
 
24
50
  def call
25
51
  @visit_ok = true
52
+ begin
53
+ expect_config!
54
+ deployment = R10K::Deployment.new(@settings)
55
+ check_write_lock!(@settings)
56
+
57
+ deployment.accept(self)
58
+ rescue => e
59
+ @visit_ok = false
60
+ logger.error R10K::Errors::Formatting.format_exception(e, @trace)
61
+ end
26
62
 
27
- expect_config!
28
- deployment = R10K::Deployment.new(@settings)
29
- check_write_lock!(@settings)
30
-
31
- deployment.accept(self)
32
63
  @visit_ok
33
64
  end
34
65
 
@@ -45,29 +76,20 @@ module R10K
45
76
  end
46
77
 
47
78
  def visit_environment(environment)
48
- if @opts[:environment] && (@opts[:environment] != environment.dirname)
49
- logger.debug1(_("Only updating modules in environment %{opt_env} skipping environment %{env_path}") % {opt_env: @opts[:environment], env_path: environment.path})
79
+ requested_envs = @settings.dig(:overrides, :environments, :requested_environments)
80
+ if !requested_envs.empty? && !requested_envs.include?(environment.dirname)
81
+ logger.debug1(_("Only updating modules in environment(s) %{opt_env} skipping environment %{env_path}") % {opt_env: requested_envs.inspect, env_path: environment.path})
50
82
  else
51
- logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @argv.inspect, env_path: environment.path})
52
- yield
53
- end
54
- end
83
+ logger.debug1(_("Updating modules %{modules} in environment %{env_path}") % {modules: @settings.dig(:overrides, :modules, :requested_modules).inspect, env_path: environment.path})
55
84
 
56
- def visit_puppetfile(puppetfile)
57
- puppetfile.load
58
- yield
59
- end
85
+ environment.deploy
60
86
 
61
- def visit_module(mod)
62
- if @argv.include?(mod.name)
63
- logger.info _("Deploying module %{mod_path}") % {mod_path: mod.path}
64
- mod.sync(force: @force)
65
- if mod.environment && @generate_types
66
- logger.debug("Generating puppet types for environment '#{mod.environment.dirname}'...")
67
- mod.environment.generate_types!
87
+ requested_mods = @settings.dig(:overrides, :modules, :requested_modules) || []
88
+ generate_types = @settings.dig(:overrides, :environments, :generate_types)
89
+ if generate_types && !((environment.modules.map(&:name) & requested_mods).empty?)
90
+ logger.debug("Generating puppet types for environment '#{environment.dirname}'...")
91
+ environment.generate_types!
68
92
  end
69
- else
70
- logger.debug1(_("Only updating modules %{modules}, skipping module %{mod_name}") % {modules: @argv.inspect, mod_name: mod.name})
71
93
  end
72
94
  end
73
95
 
@@ -77,7 +99,9 @@ module R10K
77
99
  'no-force': :self,
78
100
  'generate-types': :self,
79
101
  'puppet-path': :self,
80
- 'puppet-conf': :self)
102
+ 'puppet-conf': :self,
103
+ 'private-key': :self,
104
+ 'oauth-token': :self)
81
105
  end
82
106
  end
83
107
  end
@@ -8,7 +8,9 @@ module R10K
8
8
  class Check < R10K::Action::Base
9
9
 
10
10
  def call
11
- pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
11
+ pf = R10K::Puppetfile.new(@root,
12
+ {moduledir: @moduledir,
13
+ puppetfile_path: @puppetfile})
12
14
  begin
13
15
  pf.load!
14
16
  $stderr.puts _("Syntax OK")
@@ -2,6 +2,7 @@ require 'r10k/puppetfile'
2
2
  require 'r10k/errors/formatting'
3
3
  require 'r10k/action/visitor'
4
4
  require 'r10k/action/base'
5
+ require 'r10k/util/cleaner'
5
6
 
6
7
  module R10K
7
8
  module Action
@@ -9,33 +10,29 @@ module R10K
9
10
  class Install < R10K::Action::Base
10
11
 
11
12
  def call
12
- @visit_ok = true
13
- pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile, nil , @force)
14
- pf.accept(self)
15
- @visit_ok
16
- end
17
-
18
- private
19
-
20
- include R10K::Action::Visitor
21
-
22
- def visit_puppetfile(pf)
23
- pf.load!
24
- yield
25
- pf.purge!
26
- end
27
-
28
- def visit_module(mod)
29
- @force ||= false
30
- logger.info _("Updating module %{mod_path}") % {mod_path: mod.path}
31
-
32
- if mod.respond_to?(:desired_ref) && mod.desired_ref == :control_branch
33
- logger.warn _("Cannot track control repo branch for content '%{name}' when not part of a 'deploy' action, will use default if available." % {name: mod.name})
13
+ @ok = true
14
+ begin
15
+ pf = R10K::Puppetfile.new(@root,
16
+ {moduledir: @moduledir,
17
+ puppetfile_path: @puppetfile,
18
+ force: @force || false})
19
+ pf.load!
20
+ pf.sync
21
+
22
+ R10K::Util::Cleaner.new(pf.managed_directories,
23
+ pf.desired_contents,
24
+ pf.purge_exclusions).purge!
25
+
26
+ rescue => e
27
+ @ok = false
28
+ logger.error R10K::Errors::Formatting.format_exception(e, @trace)
34
29
  end
35
30
 
36
- mod.sync(force: @force)
31
+ @ok
37
32
  end
38
33
 
34
+ private
35
+
39
36
  def allowed_initialize_opts
40
37
  super.merge(root: :self, puppetfile: :self, moduledir: :self, force: :self )
41
38
  end
@@ -1,4 +1,5 @@
1
1
  require 'r10k/puppetfile'
2
+ require 'r10k/util/cleaner'
2
3
  require 'r10k/action/base'
3
4
  require 'r10k/errors/formatting'
4
5
 
@@ -8,9 +9,14 @@ module R10K
8
9
  class Purge < R10K::Action::Base
9
10
 
10
11
  def call
11
- pf = R10K::Puppetfile.new(@root, @moduledir, @puppetfile)
12
+ pf = R10K::Puppetfile.new(@root,
13
+ {moduledir: @moduledir,
14
+ puppetfile_path: @puppetfile})
12
15
  pf.load!
13
- pf.purge!
16
+ R10K::Util::Cleaner.new(pf.managed_directories,
17
+ pf.desired_contents,
18
+ pf.purge_exclusions).purge!
19
+
14
20
  true
15
21
  rescue => e
16
22
  logger.error R10K::Errors::Formatting.format_exception(e, @trace)
@@ -55,6 +55,10 @@ module R10K
55
55
  newval
56
56
  end
57
57
 
58
+ # Credentials from the CLI override both the global and per-repo
59
+ # credentials from the config, and so need to be handled specially
60
+ with_overrides = add_credential_overrides(with_overrides)
61
+
58
62
  @settings = R10K::Settings.global_settings.evaluate(with_overrides)
59
63
 
60
64
  R10K::Initializers::GlobalInitializer.new(@settings).call
@@ -92,6 +96,35 @@ module R10K
92
96
 
93
97
  results
94
98
  end
99
+
100
+ def add_credential_overrides(overrides)
101
+ sshkey_path = @opts[:'private-key']
102
+ token_path = @opts[:'oauth-token']
103
+
104
+ if sshkey_path && token_path
105
+ raise R10K::Error, "Cannot specify both an SSH key and a token to use with this deploy."
106
+ end
107
+
108
+ if sshkey_path
109
+ overrides[:git] ||= {}
110
+ overrides[:git][:private_key] = sshkey_path
111
+ if repo_settings = overrides[:git][:repositories]
112
+ repo_settings.each do |repo|
113
+ repo[:private_key] = sshkey_path
114
+ end
115
+ end
116
+ elsif token_path
117
+ overrides[:git] ||= {}
118
+ overrides[:git][:oauth_token] = token_path
119
+ if repo_settings = overrides[:git][:repositories]
120
+ repo_settings.each do |repo|
121
+ repo[:oauth_token] = token_path
122
+ end
123
+ end
124
+ end
125
+
126
+ overrides
127
+ end
95
128
  end
96
129
  end
97
130
  end
@@ -21,7 +21,7 @@ module R10K::CLI
21
21
  (https://puppet.com/docs/puppet/latest/environments_about.html).
22
22
  DESCRIPTION
23
23
 
24
- required nil, :cachedir, 'Specify a cachedir, overriding the value in config'
24
+ option nil, :cachedir, 'Specify a cachedir, overriding the value in config', argument: :required
25
25
  flag nil, :'no-force', 'Prevent the overwriting of local module modifications'
26
26
  flag nil, :'generate-types', 'Run `puppet generate types` after updating an environment'
27
27
  option nil, :'puppet-path', 'Path to puppet executable', argument: :required do |value, cmd|
@@ -32,6 +32,8 @@ module R10K::CLI
32
32
  end
33
33
  end
34
34
  option nil, :'puppet-conf', 'Path to puppet.conf', argument: :required
35
+ option nil, :'private-key', 'Path to SSH key to use when cloning. Only valid with rugged provider', argument: :required
36
+ option nil, :'oauth-token', 'Path to OAuth token to use when cloning. Only valid with rugged provider', argument: :required
35
37
 
36
38
  run do |opts, args, cmd|
37
39
  puts cmd.help(:verbose => opts[:verbose])
@@ -53,7 +55,7 @@ branches.
53
55
 
54
56
  Environments can provide a Puppetfile at the root of the directory to deploy
55
57
  independent Puppet modules. To recursively deploy an environment, pass the
56
- `--puppetfile` flag to the command.
58
+ `--modules` flag to the command.
57
59
 
58
60
  **NOTE**: If an environment has a Puppetfile when it is instantiated a
59
61
  recursive update will be forced. It is assumed that environments are dependent
@@ -61,8 +63,10 @@ on modules specified in the Puppetfile and an update will be automatically
61
63
  scheduled. On subsequent deployments, Puppetfile deployment will default to off.
62
64
  DESCRIPTION
63
65
 
64
- flag :p, :puppetfile, 'Deploy modules from a puppetfile'
65
- required nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile'
66
+ flag :p, :puppetfile, 'Deploy modules (deprecated, use -m)'
67
+ flag :m, :modules, 'Deploy modules'
68
+ option nil, :'default-branch-override', 'Specify a branchname to override the default branch in the puppetfile',
69
+ argument: :required
66
70
 
67
71
  runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Environment)
68
72
  end
@@ -82,7 +86,7 @@ It will load the Puppetfile configurations out of all environments, and will
82
86
  try to deploy the given module names in all environments.
83
87
  DESCRIPTION
84
88
 
85
- required :e, :environment, 'Update the modules in the given environment'
89
+ option :e, :environment, 'Update the modules in the given environment', argument: :required
86
90
 
87
91
  runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Module)
88
92
  end
@@ -97,10 +101,12 @@ try to deploy the given module names in all environments.
97
101
  usage 'display'
98
102
  summary 'Display environments and modules in the deployment'
99
103
 
100
- flag :p, :puppetfile, 'Display Puppetfile modules'
104
+ flag :p, :puppetfile, 'Display modules (deprecated, use -m)'
105
+ flag :m, :modules, 'Display modules'
101
106
  flag nil, :detail, 'Display detailed information'
102
107
  flag nil, :fetch, 'Update available environment lists from all remote sources'
103
- required nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml'
108
+ option nil, :format, 'Display output in a specific format. Valid values: json, yaml. Default: yaml',
109
+ argument: :required
104
110
 
105
111
  runner R10K::Action::CriRunner.wrap(R10K::Action::Deploy::Display)
106
112
  end
@@ -30,8 +30,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
30
30
  name 'install'
31
31
  usage 'install'
32
32
  summary 'Install all modules from a Puppetfile'
33
- required nil, :moduledir, 'Path to install modules to'
34
- required nil, :puppetfile, 'Path to puppetfile'
33
+ option nil, :moduledir, 'Path to install modules to', argument: :required
34
+ option nil, :puppetfile, 'Path to puppetfile', argument: :required
35
35
  flag nil, :force, 'Force locally changed files to be overwritten'
36
36
  runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Install)
37
37
  end
@@ -45,7 +45,7 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
45
45
  usage 'check'
46
46
  summary 'Try and load the Puppetfile to verify the syntax is correct.'
47
47
 
48
- required nil, :puppetfile, 'Path to Puppetfile'
48
+ option nil, :puppetfile, 'Path to Puppetfile', argument: :required
49
49
  runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Check)
50
50
  end
51
51
  end
@@ -58,8 +58,8 @@ Puppetfile (http://bombasticmonkey.com/librarian-puppet/).
58
58
  usage 'purge'
59
59
  summary 'Purge unmanaged modules from a Puppetfile managed directory'
60
60
 
61
- required nil, :moduledir, 'Path to install modules to'
62
- required nil, :puppetfile, 'Path to Puppetfile'
61
+ option nil, :moduledir, 'Path to install modules to', argument: :required
62
+ option nil, :puppetfile, 'Path to Puppetfile', argument: :required
63
63
  runner R10K::Action::Puppetfile::CriRunner.wrap(R10K::Action::Puppetfile::Purge)
64
64
  end
65
65
  end
@@ -0,0 +1,83 @@
1
+ module R10K
2
+ module ContentSynchronizer
3
+
4
+ def self.serial_accept(modules, visitor, loader)
5
+ visitor.visit(:puppetfile, loader) do
6
+ serial_sync(modules)
7
+ end
8
+ end
9
+
10
+ def self.serial_sync(modules)
11
+ modules.each do |mod|
12
+ mod.sync
13
+ end
14
+ end
15
+
16
+ def self.concurrent_accept(modules, visitor, loader, pool_size, logger)
17
+ mods_queue = modules_visit_queue(modules, visitor, loader)
18
+ sync_queue(mods_queue, pool_size, logger)
19
+ end
20
+
21
+ def self.concurrent_sync(modules, pool_size, logger)
22
+ mods_queue = modules_sync_queue(modules)
23
+ sync_queue(mods_queue, pool_size, logger)
24
+ end
25
+
26
+ def self.sync_queue(mods_queue, pool_size, logger)
27
+ logger.debug _("Updating modules with %{pool_size} threads") % {pool_size: pool_size}
28
+ thread_pool = pool_size.times.map { sync_thread(mods_queue, logger) }
29
+ thread_exception = nil
30
+
31
+ # If any threads raise an exception the deployment is considered a failure.
32
+ # In that event clear the queue, wait for other threads to finish their
33
+ # current work, then re-raise the first exception caught.
34
+ begin
35
+ thread_pool.each(&:join)
36
+ rescue => e
37
+ logger.error _("Error during concurrent deploy of a module: %{message}") % {message: e.message}
38
+ mods_queue.clear
39
+ thread_exception ||= e
40
+ retry
41
+ ensure
42
+ raise thread_exception unless thread_exception.nil?
43
+ end
44
+ end
45
+
46
+ def self.modules_visit_queue(modules, visitor, loader)
47
+ Queue.new.tap do |queue|
48
+ visitor.visit(:puppetfile, loader) do
49
+ enqueue_modules(queue, modules)
50
+ end
51
+ end
52
+ end
53
+
54
+ def self.modules_sync_queue(modules)
55
+ Queue.new.tap do |queue|
56
+ enqueue_modules(queue, modules)
57
+ end
58
+ end
59
+
60
+ def self.enqueue_modules(queue, modules)
61
+ modules_by_cachedir = modules.group_by { |mod| mod.cachedir }
62
+ modules_without_vcs_cachedir = modules_by_cachedir.delete(:none) || []
63
+
64
+ modules_without_vcs_cachedir.each {|mod| queue << Array(mod) }
65
+ modules_by_cachedir.values.each {|mods| queue << mods }
66
+ end
67
+
68
+ def self.sync_thread(mods_queue, logger)
69
+ Thread.new do
70
+ begin
71
+ while mods = mods_queue.pop(true) do
72
+ mods.each { |mod| mod.sync }
73
+ end
74
+ rescue ThreadError => e
75
+ logger.debug _("Module thread %{id} exiting: %{message}") % {message: e.message, id: Thread.current.object_id}
76
+ Thread.exit
77
+ rescue => e
78
+ Thread.main.raise(e)
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
@@ -118,7 +118,7 @@ module R10K
118
118
  raise R10K::Error, _("Unable to load sources; the supplied configuration does not define the 'sources' key")
119
119
  end
120
120
  @_sources = sources.map do |(name, hash)|
121
- R10K::Source.from_hash(name, hash)
121
+ R10K::Source.from_hash(name, hash.merge({overrides: @config[:overrides]}))
122
122
  end
123
123
  end
124
124