bolt 2.24.0 → 2.28.0

data/lib/bolt/config.rb

@@ -19,7 +19,7 @@ module Bolt
   class Config
     include Bolt::Config::Options
 
-    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency, :deprecations
+    attr_reader :config_files, :logs, :data, :transports, :project, :modified_concurrency, :deprecations
 
     BOLT_CONFIG_NAME = 'bolt.yaml'
     BOLT_DEFAULTS_NAME = 'bolt-defaults.yaml'
@@ -32,16 +32,19 @@ module Bolt
     end
 
     def self.from_project(project, overrides = {})
+      logs = []
       conf = if project.project_file == project.config_file
                project.data
              else
-               Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+               c = Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+               logs << { debug: "Loaded configuration from #{project.config_file}" } if File.exist?(project.config_file)
+               c
              end
 
       data = load_defaults(project).push(
         filepath: project.config_file,
         data: conf,
-        warnings: [],
+        logs: logs,
         deprecations: []
       )
 
@@ -50,17 +53,20 @@ module Bolt
 
     def self.from_file(configfile, overrides = {})
       project = Bolt::Project.create_project(Pathname.new(configfile).expand_path.dirname)
+      logs = []
 
       conf = if project.project_file == project.config_file
                project.data
              else
-               Bolt::Util.read_yaml_hash(configfile, 'config')
+               c = Bolt::Util.read_yaml_hash(configfile, 'config')
+               logs << { debug: "Loaded configuration from #{configfile}" }
+               c
              end
 
       data = load_defaults(project).push(
-        filepath: project.config_file,
+        filepath: configfile,
         data: conf,
-        warnings: [],
+        logs: logs,
         deprecations: []
       )
 
@@ -90,13 +96,13 @@ module Bolt
     def self.load_bolt_defaults_yaml(dir)
       filepath = dir + BOLT_DEFAULTS_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
-      warnings = []
+      logs     = [{ debug: "Loaded configuration from #{filepath}" }]
 
       # Warn if 'bolt.yaml' detected in same directory.
       if File.exist?(bolt_yaml = dir + BOLT_CONFIG_NAME)
-        warnings.push(
-          msg: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
-               "will be ignored."
+        logs.push(
+          warn: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
+          "will be ignored."
         )
       end
 
@@ -105,9 +111,9 @@ module Bolt
 
       if project_config.any?
         data.reject! { |key, _| project_config.include?(key) }
-        warnings.push(
-          msg: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
-                "Project configuration should be set in 'bolt-project.yaml'."
+        logs.push(
+          warn: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
+          "Project configuration should be set in 'bolt-project.yaml'."
         )
       end
 
@@ -116,10 +122,10 @@ module Bolt
 
       if transport_config.any?
         data.reject! { |key, _| transport_config.include?(key) }
-        warnings.push(
-          msg: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
-               "Transport configuration should be set under the 'inventory-config' option or "\
-               "in 'inventory.yaml'."
+        logs.push(
+          warn: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
+          "Transport configuration should be set under the 'inventory-config' option or "\
+          "in 'inventory.yaml'."
         )
       end
 
@@ -142,7 +148,7 @@ module Bolt
         data = data.merge(data.delete('inventory-config'))
       end
 
-      { filepath: filepath, data: data, warnings: warnings, deprecations: [] }
+      { filepath: filepath, data: data, logs: logs, deprecations: [] }
     end
 
     # Loads a 'bolt.yaml' file, the legacy configuration file. There's no special munging needed
@@ -150,11 +156,12 @@ module Bolt
     def self.load_bolt_yaml(dir)
       filepath = dir + BOLT_CONFIG_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
+      logs     = [{ debug: "Loaded configuration from #{filepath}" }]
       deprecations = [{ type: 'Using bolt.yaml for system configuration',
                         msg: "Configuration file #{filepath} is deprecated and will be removed in a future version "\
                         "of Bolt. Use '#{dir + BOLT_DEFAULTS_NAME}' instead." }]
 
-      { filepath: filepath, data: data, warnings: [], deprecations: deprecations }
+      { filepath: filepath, data: data, logs: logs, deprecations: deprecations }
     end
 
     def self.load_defaults(project)
@@ -187,13 +194,13 @@ module Bolt
       unless config_data.is_a?(Array)
         config_data = [{ filepath: project.config_file,
                          data: config_data,
-                         warnings: [],
+                         logs: [],
                          deprecations: [] }]
       end
 
-      @logger       = Logging.logger[self]
+      @logger       = Bolt::Logger.logger(self)
       @project      = project
-      @warnings     = @project.warnings.dup
+      @logs         = @project.logs.dup
       @deprecations = @project.deprecations.dup
       @transports   = {}
       @config_files = []
@@ -204,7 +211,7 @@ module Bolt
         'compile-concurrency' => Etc.nprocessors,
         'concurrency'         => default_concurrency,
         'format'              => 'human',
-        'log'                 => { 'console' => {}, 'bolt-debug.log' => { 'level' => 'debug', 'append' => false } },
+        'log'                 => { 'console' => {} },
         'plugin_hooks'        => {},
         'plugins'             => {},
         'puppetdb'            => {},
@@ -213,8 +220,15 @@ module Bolt
         'transport'           => 'ssh'
       }
 
+      if project.path.directory?
+        default_data['log']['bolt-debug.log'] = {
+          'level' => 'debug',
+          'append' => false
+        }
+      end
+
       loaded_data = config_data.each_with_object([]) do |data, acc|
-        @warnings.concat(data[:warnings]) if data[:warnings].any?
+        @logs.concat(data[:logs]) if data[:logs].any?
         @deprecations.concat(data[:deprecations]) if data[:deprecations].any?
 
         if data[:data].any?
@@ -330,10 +344,26 @@ module Bolt
     end
 
     private def update_logs(logs)
+      begin
+        if logs['bolt-debug.log'] && logs['bolt-debug.log'] != 'disable'
+          FileUtils.touch(File.expand_path('bolt-debug.log', @project.path))
+        end
+      rescue StandardError
+        logs.delete('bolt-debug.log')
+      end
+
       logs.each_with_object({}) do |(key, val), acc|
-        next unless val.is_a?(Hash)
+        # Remove any disabled logs
+        next if val == 'disable'
 
         name = normalize_log(key)
+
+        # But otherwise it has to be a Hash
+        unless val.is_a?(Hash)
+          raise Bolt::ValidationError,
+                "config of log #{name} must be a Hash, received #{val.class} #{val.inspect}"
+        end
+
         acc[name] = val.slice('append', 'level')
                        .transform_keys(&:to_sym)
 
@@ -358,7 +388,7 @@ module Bolt
     def validate
       if @data['future']
         msg = "Configuration option 'future' no longer exposes future behavior."
-        @warnings << { option: 'future', msg: msg }
+        @logs << { warn: msg }
       end
 
       keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]

data/lib/bolt/config/options.rb

@@ -176,7 +176,9 @@ module Bolt
           description: "A map of configuration for the logfile output. Under `log`, you can configure log options "\
                        "for `console` and add configuration for individual log files, such as "\
                        "`~/.puppetlabs/bolt/debug.log`. Individual log files must be valid filepaths. If the log "\
-                       "file does not exist, then Bolt will create it before logging information.",
+                       "file does not exist, then Bolt will create it before logging information. Set the value to "\
+                       "`disable` to remove a log file defined at an earlier level of the config hierarchy. By "\
+                       "default, Bolt logs to a bolt-debug.log file in the Bolt project directory.",
           type: Hash,
           properties: {
             "console" => {
@@ -194,7 +196,8 @@ module Bolt
           },
           additionalProperties: {
             description: "Configuration for the logfile output.",
-            type: Hash,
+            type: [String, Hash],
+            enum: ['disable'],
             properties: {
               "append" => {
                 description: "Whether to append output to an existing log file.",
@@ -224,6 +227,24 @@ module Bolt
           _example: ["~/.puppetlabs/bolt/modules", "~/.puppetlabs/bolt/site-modules"],
           _default: ["project/modules", "project/site-modules", "project/site"]
         },
+        "modules" => {
+          description: "A list of module dependencies for the project. Each dependency is a map of data specifying "\
+                       "the module to install. To install the project's module dependencies, run the `bolt module "\
+                       "install` command.",
+          type: Array,
+          items: {
+            type: Hash,
+            required: ["name"],
+            properties: {
+              "name" => {
+                description: "The name of the module.",
+                type: String
+              }
+            }
+          },
+          _plugin: false,
+          _example: [{ "name" => "puppetlabs-mysql" }, { "name" => "puppetlabs-apache" }]
+        },
         "name" => {
           description: "The name of the Bolt project. When this option is configured, the project is considered a "\
                        "[Bolt project](experimental_features.md#bolt-projects), allowing Bolt to load content from "\
@@ -473,6 +494,7 @@ module Bolt
         inventoryfile
         log
         modulepath
+        modules
         name
         plans
         plugin_hooks

data/lib/bolt/executor.rb

@@ -40,7 +40,7 @@ module Bolt
       require 'concurrent'
 
       @analytics = analytics
-      @logger = Logging.logger[self]
+      @logger = Bolt::Logger.logger(self)
 
       @transports = Bolt::TRANSPORTS.each_with_object({}) do |(key, val), coll|
         coll[key.to_s] = if key == :remote

data/lib/bolt/inventory.rb

@@ -46,10 +46,13 @@ module Bolt
     end
 
     def self.from_config(config, plugins)
+      logger = Bolt::Logger.logger(self)
+
       if ENV.include?(ENVIRONMENT_VAR)
         begin
           data = YAML.safe_load(ENV[ENVIRONMENT_VAR])
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}" unless data.is_a?(Hash)
+          logger.debug("Loaded inventory from environment variable #{ENVIRONMENT_VAR}")
         rescue Psych::Exception
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
@@ -57,8 +60,12 @@ module Bolt
         data = if config.inventoryfile
                  Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
                else
-                 Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+                 i = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+                 logger.debug("Loaded inventory from #{config.default_inventoryfile}") if i
+                 i
                end
+        # This avoids rubocop complaining about identical conditionals
+        logger.debug("Loaded inventory from #{config.inventoryfile}") if config.inventoryfile
       end
 
       # Resolve plugin references from transport config

data/lib/bolt/inventory/group.rb

@@ -19,7 +19,7 @@ module Bolt
       CONFIG_KEYS = Bolt::Config::INVENTORY_OPTIONS.keys
 
       def initialize(input, plugins)
-        @logger = Logging.logger[self]
+        @logger = Bolt::Logger.logger(self)
         @plugins = plugins
 
         input = @plugins.resolve_top_level_references(input) if @plugins.reference?(input)

data/lib/bolt/inventory/inventory.rb

@@ -16,7 +16,7 @@ module Bolt
 
       # TODO: Pass transport config instead of config object
       def initialize(data, transport, transports, plugins)
-        @logger       = Logging.logger[self]
+        @logger       = Bolt::Logger.logger(self)
         @data         = data || {}
         @transport    = transport
         @config       = transports

data/lib/bolt/inventory/target.rb

@@ -13,7 +13,7 @@ module Bolt
           raise Bolt::Inventory::ValidationError.new("Target must have either a name or uri", nil)
         end
 
-        @logger = Logging.logger[inventory]
+        @logger = Bolt::Logger.logger(inventory)
 
         # If the target isn't mentioned by any groups, it won't have a uri or
         # name and we will use the target_name as both

data/lib/bolt/logger.rb

@@ -28,7 +28,7 @@ module Bolt
     end
 
     def self.configure(destinations, color)
-      root_logger = Logging.logger[:root]
+      root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
         'console',
@@ -66,6 +66,13 @@ module Bolt
       end
     end
 
+    # A helper to ensure the Logging library is always initialized with our
+    # custom log levels before retrieving a Logger instance.
+    def self.logger(name)
+      initialize_logging
+      Logging.logger[name]
+    end
+
     def self.analytics=(analytics)
       @analytics = analytics
     end
@@ -110,7 +117,7 @@ module Bolt
     def self.warn_once(type, msg)
       @mutex.synchronize {
         @warnings ||= []
-        @logger ||= Logging.logger[self]
+        @logger ||= Bolt::Logger.logger(self)
         unless @warnings.include?(type)
           @logger.warn(msg)
           @warnings << type

data/lib/bolt/outputter/logger.rb

@@ -7,7 +7,7 @@ module Bolt
     class Logger < Bolt::Outputter
       def initialize(verbose, trace)
         super(false, verbose, trace)
-        @logger = Logging.logger[self]
+        @logger = Bolt::Logger.logger(self)
       end
 
       def handle_event(event)

data/lib/bolt/pal.rb

@@ -65,7 +65,7 @@ module Bolt
       @resource_types = resource_types
       @project = project
 
-      @logger = Logging.logger[self]
+      @logger = Bolt::Logger.logger(self)
       if modulepath && !modulepath.empty?
         @logger.debug("Loading modules from #{@modulepath.join(File::PATH_SEPARATOR)}")
       end
@@ -76,7 +76,7 @@ module Bolt
     # Puppet logging is global so this is class method to avoid confusion
     def self.configure_logging
       Puppet::Util::Log.destinations.clear
-      Puppet::Util::Log.newdestination(Logging.logger['Puppet'])
+      Puppet::Util::Log.newdestination(Bolt::Logger.logger('Puppet'))
       # Defer all log level decisions to the Logging library by telling Puppet
       # to log everything
       Puppet.settings[:log_level] = 'debug'
@@ -141,6 +141,19 @@ module Bolt
       end
     end
 
+    def detect_project_conflict(project, environment)
+      return unless project && project.load_as_module?
+      # The environment modulepath has stripped out non-existent directories,
+      # so we don't need to check for them
+      modules = environment.modulepath.flat_map do |path|
+        Dir.children(path).select { |name| Puppet::Module.is_module_directory?(name, path) }
+      end
+      if modules.include?(project.name)
+        Bolt::Logger.warn_once("project shadows module",
+                               "The project '#{project.name}' shadows an existing module of the same name")
+      end
+    end
+
     # Runs a block in a PAL script compiler configured for Bolt.  Catches
     # exceptions thrown by the block and re-raises them ensuring they are
     # Bolt::Errors since the script compiler block will squash all exceptions.
@@ -149,15 +162,13 @@ module Bolt
       setup
       r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
-        bolt_project = @project if @project&.name
-        # Puppet currently won't receive the project unless it is a named project. Since
-        # the download_file plan function needs access to the project path, add it to the
-        # context.
-        bolt_project_data = @project
-        Puppet.override(bolt_project: bolt_project,
-                        bolt_project_data: bolt_project_data,
+        Puppet.override(bolt_project: @project,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
-          pal.with_script_compiler do |compiler|
+          # Because this has the side effect of loading and caching the list
+          # of modules, it must happen *after* we have overridden
+          # bolt_project or the project will be ignored
+          detect_project_conflict(@project, Puppet.lookup(:environments).get('bolt'))
+          pal.with_script_compiler(set_local_facts: false) do |compiler|
             alias_types(compiler)
             register_resource_types(Puppet.lookup(:loaders)) if @resource_types
             begin

data/lib/bolt/pal/yaml_plan/evaluator.rb

@@ -7,7 +7,7 @@ module Bolt
     class YamlPlan
       class Evaluator
         def initialize(analytics = Bolt::Analytics::NoopClient.new)
-          @logger = Logging.logger[self]
+          @logger = Bolt::Logger.logger(self)
           @analytics = analytics
           @evaluator = Puppet::Pops::Parser::EvaluatingParser.new
         end

data/lib/bolt/plugin/puppetdb.rb

@@ -19,7 +19,7 @@ module Bolt
       def initialize(config:, context:)
         pdb_config = Bolt::PuppetDB::Config.load_config(config, context.boltdir)
         @puppetdb_client = Bolt::PuppetDB::Client.new(pdb_config)
-        @logger = Logging.logger[self]
+        @logger = Bolt::Logger.logger(self)
       end
 
       def name

data/lib/bolt/project.rb

@@ -17,37 +17,53 @@ module Bolt
     }.freeze
 
     attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
-                :puppetfile, :rerunfile, :type, :resource_types, :warnings, :project_file,
+                :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
                 :deprecations, :downloads, :plans_path
 
-    def self.default_project
-      create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user')
+    def self.default_project(logs = [])
+      create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
     # If homedir isn't defined use the system config path
     rescue ArgumentError
-      create_project(Bolt::Config.system_path, 'system')
+      create_project(Bolt::Config.system_path, 'system', logs)
     end
 
     # Search recursively up the directory hierarchy for the Project. Look for a
     # directory called Boltdir or a file called bolt.yaml (for a control repo
     # type Project). Otherwise, repeat the check on each directory up the
     # hierarchy, falling back to the default if we reach the root.
-    def self.find_boltdir(dir)
+    def self.find_boltdir(dir, logs = [])
       dir = Pathname.new(dir)
 
       if (dir + BOLTDIR_NAME).directory?
-        create_project(dir + BOLTDIR_NAME, 'embedded')
+        create_project(dir + BOLTDIR_NAME, 'embedded', logs)
       elsif (dir + 'bolt.yaml').file? || (dir + 'bolt-project.yaml').file?
-        create_project(dir, 'local')
+        create_project(dir, 'local', logs)
       elsif dir.root?
-        default_project
+        default_project(logs)
       else
-        find_boltdir(dir.parent)
+        logs << { debug: "Did not detect Boltdir, bolt.yaml, or bolt-project.yaml at '#{dir}'. "\
+                  "This directory won't be loaded as a project." }
+        find_boltdir(dir.parent, logs)
       end
     end
 
-    def self.create_project(path, type = 'option')
+    def self.create_project(path, type = 'option', logs = [])
       fullpath = Pathname.new(path).expand_path
 
+      if type == 'user'
+        begin
+          # This is already expanded if the type is user
+          FileUtils.mkdir_p(path)
+        rescue StandardError
+          logs << { warn: "Could not create default project at #{path}. Continuing without a writeable project. "\
+                    "Log and rerun files will not be written." }
+        end
+      end
+
+      if type == 'option' && !File.directory?(path)
+        raise Bolt::Error.new("Could not find project at #{path}", "bolt/project-error")
+      end
+
       if !Bolt::Util.windows? && type != 'environment' && fullpath.world_writable?
         raise Bolt::Error.new(
           "Project directory '#{fullpath}' is world-writable which poses a security risk. Set "\
@@ -58,15 +74,18 @@ module Bolt
 
       project_file = File.join(fullpath, 'bolt-project.yaml')
       data = Bolt::Util.read_optional_yaml_hash(File.expand_path(project_file), 'project')
-      new(data, path, type)
+      default = type =~ /user|system/ ? 'default ' : ''
+      exist = File.exist?(File.expand_path(project_file))
+      logs << { info: "Loaded #{default}project from '#{fullpath}'" } if exist
+      new(data, path, type, logs)
     end
 
-    def initialize(raw_data, path, type = 'option')
+    def initialize(raw_data, path, type = 'option', logs = [])
       @path = Pathname.new(path).expand_path
 
       @project_file = @path + 'bolt-project.yaml'
 
-      @warnings = []
+      @logs = logs
       @deprecations = []
       if (@path + 'bolt.yaml').file? && project_file?
         msg = "Project-level configuration in bolt.yaml is deprecated if using bolt-project.yaml. "\
@@ -88,7 +107,7 @@ module Bolt
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
         msg = "Transport configuration isn't supported in bolt-project.yaml. Ignoring keys #{tc}"
-        @warnings << { msg: msg }
+        @logs << { warn: msg }
       end
 
       @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_OPTIONS.include?(k) }
@@ -98,7 +117,7 @@ module Bolt
       @config_file = if (Bolt::Config::BOLT_OPTIONS & @data.keys).any?
                        if (@path + 'bolt.yaml').file?
                          msg = "bolt-project.yaml contains valid config keys, bolt.yaml will be ignored"
-                         @warnings << { msg: msg }
+                         @logs << { warn: msg }
                        end
                        @project_file
                      else
@@ -114,7 +133,9 @@ module Bolt
     # This API is used to prepend the project as a module to Puppet's internal
     # module_references list. CHANGE AT YOUR OWN RISK
     def to_h
-      { path: @path.to_s, name: name }
+      { path: @path.to_s,
+        name: name,
+        load_as_module?: load_as_module? }
     end
 
     def eql?(other)
@@ -126,6 +147,10 @@ module Bolt
       @project_file.file?
     end
 
+    def load_as_module?
+      !name.nil?
+    end
+
     def name
       @data['name']
     end
@@ -138,6 +163,10 @@ module Bolt
       @data['plans']
     end
 
+    def modules
+      @data['modules']
+    end
+
     def validate
       if name
         if name !~ Bolt::Module::MODULE_NAME_REGEX
@@ -151,7 +180,7 @@ module Bolt
         end
       else
         message = "No project name is specified in bolt-project.yaml. Project-level content will not be available."
-        @warnings << { msg: message }
+        @logs << { warn: message }
       end
 
       %w[tasks plans].each do |conf|
@@ -159,6 +188,23 @@ module Bolt
           raise Bolt::ValidationError, "'#{conf}' in bolt-project.yaml must be an array"
         end
       end
+
+      if @data['modules']
+        unless @data['modules'].is_a?(Array)
+          raise Bolt::ValidationError, "'modules' in bolt-project.yaml must be an array"
+        end
+
+        @data['modules'].each do |mod|
+          next if mod.is_a?(Hash)
+          raise Bolt::ValidationError, "Module declaration #{mod.inspect} must be a hash"
+        end
+
+        unknown_keys = data['modules'].flat_map(&:keys).uniq - ['name']
+        if unknown_keys.any?
+          @logs << { warn: "Module declarations in bolt-project.yaml only support a name key. Ignoring "\
+                           "unsupported keys: #{unknown_keys.join(', ')}." }
+        end
+      end
     end
 
     def check_deprecated_file