bolt 2.12.0 → 2.17.0

data/lib/bolt/cli.rb

@@ -115,7 +115,12 @@ module Bolt
                   Bolt::Config.from_file(options[:configfile], options)
                 else
                   project = if options[:boltdir]
-                              Bolt::Project.new(options[:boltdir])
+                              dir = Pathname.new(options[:boltdir])
+                              if (dir + Bolt::Project::BOLTDIR_NAME).directory?
+                                Bolt::Project.create_project(dir + Bolt::Project::BOLTDIR_NAME)
+                              else
+                                Bolt::Project.create_project(dir)
+                              end
                             else
                               Bolt::Project.find_boltdir(Dir.pwd)
                             end
@@ -219,7 +224,7 @@ module Bolt
       end
 
       if options[:boltdir] && options[:configfile]
-        raise Bolt::CLIError, "Only one of '--boltdir' or '--configfile' may be specified"
+        raise Bolt::CLIError, "Only one of '--boltdir', '--project', or '--configfile' may be specified"
       end
 
       if options[:noop] &&
@@ -245,6 +250,10 @@ module Bolt
          !options[:object]
         raise Bolt::CLIError, "Must specify a value to #{options[:action]}"
       end
+
+      if options.key?(:debug) && options.key?(:log)
+        raise Bolt::CLIError, "Only one of '--debug' or '--log-level' may be specified"
+      end
     end
 
     def handle_parser_errors
@@ -392,7 +401,7 @@ module Bolt
         end
         code = apply_manifest(options[:code], options[:targets], options[:object], options[:noop])
       else
-        executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop])
+        executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop], config.modified_concurrency)
         targets = options[:targets]
 
         results = nil
@@ -512,8 +521,8 @@ module Bolt
                        params: plan_arguments }
       plan_context[:description] = options[:description] if options[:description]
 
-      executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop])
-      if options.fetch(:format, 'human') == 'human'
+      executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop], config.modified_concurrency)
+      if %w[human rainbow].include?(options.fetch(:format, 'human'))
         executor.subscribe(outputter)
       else
         # Only subscribe to out::message events for JSON outputter
@@ -548,7 +557,7 @@ module Bolt
         @logger.warn(message)
       end
 
-      executor = Bolt::Executor.new(config.concurrency, analytics, noop)
+      executor = Bolt::Executor.new(config.concurrency, analytics, noop, config.modified_concurrency)
       executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
       executor.subscribe(log_outputter)
       # apply logging looks like plan logging, so tell the outputter we're in a
@@ -771,14 +780,13 @@ module Bolt
     end
 
     def pal
-      project = config.project.load_as_module? ? config.project : nil
       @pal ||= Bolt::PAL.new(config.modulepath,
                              config.hiera_config,
                              config.project.resource_types,
                              config.compile_concurrency,
                              config.trusted_external,
                              config.apply_settings,
-                             project)
+                             config.project)
     end
 
     def convert_plan(plan)

data/lib/bolt/config.rb

@@ -6,13 +6,7 @@ require 'pathname'
 require 'bolt/project'
 require 'bolt/logger'
 require 'bolt/util'
-# Transport config objects
-require 'bolt/config/transport/ssh'
-require 'bolt/config/transport/winrm'
-require 'bolt/config/transport/orch'
-require 'bolt/config/transport/local'
-require 'bolt/config/transport/docker'
-require 'bolt/config/transport/remote'
+require 'bolt/config/options'
 
 module Bolt
   class UnknownTransportError < Bolt::Error
@@ -23,146 +17,165 @@ module Bolt
   end
 
   class Config
-    attr_reader :config_files, :warnings, :data, :transports, :project
-
-    TRANSPORT_CONFIG = {
-      'ssh'    => Bolt::Config::Transport::SSH,
-      'winrm'  => Bolt::Config::Transport::WinRM,
-      'pcp'    => Bolt::Config::Transport::Orch,
-      'local'  => Bolt::Config::Transport::Local,
-      'docker' => Bolt::Config::Transport::Docker,
-      'remote' => Bolt::Config::Transport::Remote
-    }.freeze
-
-    # NOTE: All configuration options should have a corresponding schema property
-    #       in schemas/bolt-config.schema.json
-    OPTIONS = {
-      "apply_settings"           => "A map of Puppet settings to use when applying Puppet code",
-      "color"                    => "Whether to use colored output when printing messages to the console.",
-      "compile-concurrency"      => "The maximum number of simultaneous manifest block compiles.",
-      "concurrency"              => "The number of threads to use when executing on remote targets.",
-      "format"                   => "The format to use when printing results. Options are `human` and `json`.",
-      "hiera-config"             => "The path to your Hiera config.",
-      "inventoryfile"            => "The path to a structured data inventory file used to refer to groups of "\
-                                    "targets on the command line and from plans.",
-      "log"                      => "The configuration of the logfile output. Configuration can be set for "\
-                                    "`console` and the path to a log file, such as `~/.puppetlabs/bolt/debug.log`.",
-      "modulepath"               => "The module path for loading tasks and plan code. This is either an array "\
-                                    "of directories or a string containing a list of directories separated by the "\
-                                    "OS-specific PATH separator.",
-      "plugin_hooks"             => "Which plugins a specific hook should use.",
-      "plugins"                  => "A map of plugins and their configuration data.",
-      "puppetdb"                 => "A map containing options for configuring the Bolt PuppetDB client.",
-      "puppetfile"               => "A map containing options for the `bolt puppetfile install` command.",
-      "save-rerun"               => "Whether to update `.rerun.json` in the Bolt project directory. If "\
-                                    "your target names include passwords, set this value to `false` to avoid "\
-                                    "writing passwords to disk.",
-      "transport"                => "The default transport to use when the transport for a target is not "\
-                                    "specified in the URL or inventory.",
-      "trusted-external-command" => "The path to an executable on the Bolt controller that can produce "\
-                                    "external trusted facts. **External trusted facts are experimental in both "\
-                                    "Puppet and Bolt and this API may change or be removed.**"
-    }.freeze
-
-    DEFAULT_OPTIONS = {
-      "color" => true,
-      "compile-concurrency" => "Number of cores",
-      "concurrency" => "100 or one-third of the ulimit, whichever is lower",
-      "format" => "human",
-      "hiera-config" => "Boltdir/hiera.yaml",
-      "inventoryfile" => "Boltdir/inventory.yaml",
-      "modulepath" => ["Boltdir/modules", "Boltdir/site-modules", "Boltdir/site"],
-      "save-rerun" => true
-    }.freeze
-
-    PUPPETFILE_OPTIONS = {
-      "forge" => "A subsection that can have its own `proxy` setting to set an HTTP proxy for Forge operations "\
-                 "only, and a `baseurl` setting to specify a different Forge host.",
-      "proxy" => "The HTTP proxy to use for Git and Forge operations."
-    }.freeze
-
-    LOG_OPTIONS = {
-      "append" => "Add output to an existing log file. Available only for logs output to a "\
-                  "filepath.",
-      "level"  => "The type of information in the log. Either `debug`, `info`, `notice`, "\
-                  "`warn`, or `error`."
-    }.freeze
-
-    DEFAULT_LOG_OPTIONS = {
-      "append" => true,
-      "level"  => "`warn` for console, `notice` for file"
-    }.freeze
-
-    APPLY_SETTINGS = {
-      "show_diff" => "Whether to log and report a contextual diff when files are being replaced. "\
-                     "See [Puppet documentation](https://puppet.com/docs/puppet/latest/configuration.html#showdiff) "\
-                     "for details"
-    }.freeze
-
-    DEFAULT_APPLY_SETTINGS = {
-      "show_diff" => false
-    }.freeze
+    include Bolt::Config::Options
 
+    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency
+
+    BOLT_CONFIG_NAME = 'bolt.yaml'
+    BOLT_DEFAULTS_NAME = 'bolt-defaults.yaml'
+
+    # The default concurrency value that is used when the ulimit is not low (i.e. < 700)
     DEFAULT_DEFAULT_CONCURRENCY = 100
 
     def self.default
-      new(Bolt::Project.new('.'), {})
+      new(Bolt::Project.create_project('.'), {})
     end
 
     def self.from_project(project, overrides = {})
-      data = {
-        filepath: project.config_file,
-        data: Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
-      }
+      conf = if project.project_file == project.config_file
+               project.data
+             else
+               Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+             end
 
-      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
+      data = load_defaults(project).push(
+        filepath: project.config_file,
+        data: conf,
+        warnings: []
+      )
 
       new(project, data, overrides)
     end
 
     def self.from_file(configfile, overrides = {})
-      project = Bolt::Project.new(Pathname.new(configfile).expand_path.dirname)
+      project = Bolt::Project.create_project(Pathname.new(configfile).expand_path.dirname)
 
-      data = {
+      conf = if project.project_file == project.config_file
+               project.data
+             else
+               Bolt::Util.read_yaml_hash(configfile, 'config')
+             end
+
+      data = load_defaults(project).push(
         filepath: project.config_file,
-        data: Bolt::Util.read_yaml_hash(configfile, 'config')
-      }
-      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
+        data: conf,
+        warnings: []
+      )
 
       new(project, data, overrides)
     end
 
-    def self.load_defaults(project)
+    def self.system_path
       # Lazy-load expensive gem code
       require 'win32/dir' if Bolt::Util.windows?
 
-      # Don't load /etc/puppetlabs/bolt/bolt.yaml twice
-      confs = if project.path == Bolt::Project.system_path
-                []
-              else
-                system_path = Pathname.new(File.join(Bolt::Project.system_path, 'bolt.yaml'))
-                [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
-              end
-
-      user_path = begin
-                    Pathname.new(File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt', 'bolt.yaml')))
-                  rescue ArgumentError
-                    nil
-                  end
-
-      confs << { filepath: user_path, data: Bolt::Util.read_optional_yaml_hash(user_path, 'config') } if user_path
+      if Bolt::Util.windows?
+        Pathname.new(File.join(Dir::COMMON_APPDATA, 'PuppetLabs', 'bolt', 'etc'))
+      else
+        Pathname.new(File.join('/etc', 'puppetlabs', 'bolt'))
+      end
+    end
+
+    def self.user_path
+      Pathname.new(File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt')))
+    rescue StandardError
+      nil
+    end
+
+    # Loads a 'bolt-defaults.yaml' file, which contains default configuration that applies to all
+    # projects. This file does not allow project-specific configuration such as 'hiera-config' and
+    # 'inventoryfile', and nests all default inventory configuration under an 'inventory-config' key.
+    def self.load_bolt_defaults_yaml(dir)
+      filepath = dir + BOLT_DEFAULTS_NAME
+      data     = Bolt::Util.read_yaml_hash(filepath, 'config')
+      warnings = []
+
+      # Warn if 'bolt.yaml' detected in same directory.
+      if File.exist?(bolt_yaml = dir + BOLT_CONFIG_NAME)
+        warnings.push(
+          msg: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
+               "will be ignored."
+        )
+      end
+
+      # Remove project-specific config such as hiera-config, etc.
+      project_config = data.slice(*(BOLT_PROJECT_OPTIONS - BOLT_DEFAULTS_OPTIONS))
+
+      if project_config.any?
+        data.reject! { |key, _| project_config.include?(key) }
+        warnings.push(
+          msg: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
+                "Project configuration should be set in 'bolt-project.yaml'."
+        )
+      end
+
+      # Remove top-level transport config such as transport, ssh, etc.
+      transport_config = data.slice(*INVENTORY_OPTIONS.keys)
+
+      if transport_config.any?
+        data.reject! { |key, _| transport_config.include?(key) }
+        warnings.push(
+          msg: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
+               "Transport configuration should be set under the 'inventory-config' option or "\
+               "in 'inventory.yaml'."
+        )
+      end
+
+      # Move data under transport-config to top-level so it can be easily merged with
+      # config from other sources.
+      if data.key?('inventory-config')
+        data = data.merge(data.delete('inventory-config'))
+      end
+
+      { filepath: filepath, data: data, warnings: warnings }
+    end
+
+    # Loads a 'bolt.yaml' file, the legacy configuration file. There's no special munging needed
+    # here since Bolt::Config will just ignore any invalid keys.
+    def self.load_bolt_yaml(dir)
+      filepath = dir + BOLT_CONFIG_NAME
+      data     = Bolt::Util.read_yaml_hash(filepath, 'config')
+      warnings = [msg: "Configuration file #{filepath} is deprecated and will be removed in a future version "\
+                        "of Bolt. Use '#{dir + BOLT_DEFAULTS_NAME}' instead."]
+
+      { filepath: filepath, data: data, warnings: warnings }
+    end
+
+    def self.load_defaults(project)
+      confs = []
+
+      # Load system-level config. Prefer a 'bolt-defaults.yaml' file, but fall back to the
+      # legacy 'bolt.yaml' file. If the project-level config file is also the system-level
+      # config file, don't load it a second time.
+      if File.exist?(system_path + BOLT_DEFAULTS_NAME)
+        confs << load_bolt_defaults_yaml(system_path)
+      elsif File.exist?(system_path + BOLT_CONFIG_NAME) &&
+            (system_path + BOLT_CONFIG_NAME) != project.config_file
+        confs << load_bolt_yaml(system_path)
+      end
+
+      # Load user-level config if there is a homedir. Prefer a 'bolt-defaults.yaml' file, but
+      # fall back to the legacy 'bolt.yaml' file.
+      if user_path
+        if File.exist?(user_path + BOLT_DEFAULTS_NAME)
+          confs << load_bolt_defaults_yaml(user_path)
+        elsif File.exist?(user_path + BOLT_CONFIG_NAME)
+          confs << load_bolt_yaml(user_path)
+        end
+      end
+
       confs
     end
 
     def initialize(project, config_data, overrides = {})
       unless config_data.is_a?(Array)
-        config_data = [{ filepath: project.config_file, data: config_data }]
+        config_data = [{ filepath: project.config_file, data: config_data, warnings: [] }]
       end
 
-      @logger = Logging.logger[self]
-      @warnings = []
-      @project = project
-      @transports = {}
+      @logger       = Logging.logger[self]
+      @project      = project
+      @warnings     = @project.warnings.dup
+      @transports   = {}
       @config_files = []
 
       default_data = {
@@ -180,24 +193,22 @@ module Bolt
         'transport'           => 'ssh'
       }
 
-      loaded_data = config_data.map do |config|
-        @config_files.push(config[:filepath])
-        config[:data]
+      loaded_data = config_data.each_with_object([]) do |data, acc|
+        @warnings.concat(data[:warnings]) if data[:warnings].any?
+
+        if data[:data].any?
+          @config_files.push(data[:filepath])
+          acc.push(data[:data])
+        end
       end
 
       override_data = normalize_overrides(overrides)
 
       # If we need to lower concurrency and concurrency is not configured
       ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
-      if default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
-         !ld_concurrency &&
-         !override_data.key?('concurrency')
-        concurrency_warning = { option: 'concurrency',
-                                msg: "Concurrency will default to #{default_concurrency} because ulimit "\
-                                "is low: #{Etc.sysconf(Etc::SC_OPEN_MAX)}. Set concurrency with "\
-                                "'--concurrency', or set your ulimit with 'ulimit -n <limit>'" }
-        @warnings << concurrency_warning
-      end
+      @modified_concurrency = default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+                              !ld_concurrency &&
+                              !override_data.key?('concurrency')
 
       @data = merge_config_layers(default_data, *loaded_data, override_data)
 
@@ -214,12 +225,13 @@ module Bolt
     def normalize_overrides(options)
       opts = options.transform_keys(&:to_s)
 
-      # Pull out config options
-      overrides = opts.slice(*OPTIONS.keys)
+      # Pull out config options. We need to add 'transport' as it's not part of the
+      # OPTIONS hash but is a valid option that can be set with the --transport CLI option
+      overrides = opts.slice(*OPTIONS.keys, 'transport')
 
       # Pull out transport config options
       TRANSPORT_CONFIG.each do |transport, config|
-        overrides[transport] = opts.slice(*config.options.keys)
+        overrides[transport] = opts.slice(*config.options)
       end
 
       # Set console log to debug if in debug mode
@@ -286,8 +298,8 @@ module Bolt
       end
 
       # Filter hashes to only include valid options
-      @data['apply_settings'] = @data['apply_settings'].slice(*APPLY_SETTINGS.keys)
-      @data['puppetfile'] = @data['puppetfile'].slice(*PUPPETFILE_OPTIONS.keys)
+      @data['apply_settings'] = @data['apply_settings'].slice(*OPTIONS['apply_settings'][:properties].keys)
+      @data['puppetfile'] = @data['puppetfile'].slice(*OPTIONS['puppetfile'][:properties].keys)
     end
 
     private def normalize_log(target)
@@ -301,7 +313,7 @@ module Bolt
         next unless val.is_a?(Hash)
 
         name = normalize_log(key)
-        acc[name] = val.slice(*LOG_OPTIONS.keys)
+        acc[name] = val.slice('append', 'level')
                        .transform_keys(&:to_sym)
 
         if (v = acc[name][:level])
@@ -353,7 +365,7 @@ module Bolt
         raise Bolt::ValidationError, "Compilation is CPU-intensive, set concurrency less than #{compile_limit}"
       end
 
-      unless %w[human json].include? format
+      if (format == 'rainbow' && Bolt::Util.windows?) || !(%w[human json rainbow].include? format)
         raise Bolt::ValidationError, "Unsupported format: '#{format}'"
       end
 
@@ -485,7 +497,7 @@ module Bolt
       @default_concurrency ||= if !sc_open_max_available? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300
                                  DEFAULT_DEFAULT_CONCURRENCY
                                else
-                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 3
+                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 7
                                end
     end
   end