bolt 2.24.1 → 2.29.0

data/lib/bolt/config.rb

@@ -19,7 +19,7 @@ module Bolt
   class Config
     include Bolt::Config::Options
 
-    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency, :deprecations
+    attr_reader :config_files, :logs, :data, :transports, :project, :modified_concurrency, :deprecations
 
     BOLT_CONFIG_NAME = 'bolt.yaml'
     BOLT_DEFAULTS_NAME = 'bolt-defaults.yaml'
@@ -32,16 +32,19 @@ module Bolt
     end
 
     def self.from_project(project, overrides = {})
+      logs = []
       conf = if project.project_file == project.config_file
                project.data
              else
-               Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+               c = Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+               logs << { debug: "Loaded configuration from #{project.config_file}" } if File.exist?(project.config_file)
+               c
              end
 
       data = load_defaults(project).push(
         filepath: project.config_file,
         data: conf,
-        warnings: [],
+        logs: logs,
         deprecations: []
       )
 
@@ -50,17 +53,20 @@ module Bolt
 
     def self.from_file(configfile, overrides = {})
       project = Bolt::Project.create_project(Pathname.new(configfile).expand_path.dirname)
+      logs = []
 
       conf = if project.project_file == project.config_file
                project.data
              else
-               Bolt::Util.read_yaml_hash(configfile, 'config')
+               c = Bolt::Util.read_yaml_hash(configfile, 'config')
+               logs << { debug: "Loaded configuration from #{configfile}" }
+               c
              end
 
       data = load_defaults(project).push(
-        filepath: project.config_file,
+        filepath: configfile,
         data: conf,
-        warnings: [],
+        logs: logs,
         deprecations: []
       )
 
@@ -90,13 +96,13 @@ module Bolt
     def self.load_bolt_defaults_yaml(dir)
       filepath = dir + BOLT_DEFAULTS_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
-      warnings = []
+      logs     = [{ debug: "Loaded configuration from #{filepath}" }]
 
       # Warn if 'bolt.yaml' detected in same directory.
       if File.exist?(bolt_yaml = dir + BOLT_CONFIG_NAME)
-        warnings.push(
-          msg: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
-               "will be ignored."
+        logs.push(
+          warn: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
+          "will be ignored."
         )
       end
 
@@ -105,9 +111,9 @@ module Bolt
 
       if project_config.any?
         data.reject! { |key, _| project_config.include?(key) }
-        warnings.push(
-          msg: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
-                "Project configuration should be set in 'bolt-project.yaml'."
+        logs.push(
+          warn: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
+          "Project configuration should be set in 'bolt-project.yaml'."
         )
       end
 
@@ -116,10 +122,10 @@ module Bolt
 
       if transport_config.any?
         data.reject! { |key, _| transport_config.include?(key) }
-        warnings.push(
-          msg: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
-               "Transport configuration should be set under the 'inventory-config' option or "\
-               "in 'inventory.yaml'."
+        logs.push(
+          warn: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
+          "Transport configuration should be set under the 'inventory-config' option or "\
+          "in 'inventory.yaml'."
         )
       end
 
@@ -142,7 +148,7 @@ module Bolt
         data = data.merge(data.delete('inventory-config'))
       end
 
-      { filepath: filepath, data: data, warnings: warnings, deprecations: [] }
+      { filepath: filepath, data: data, logs: logs, deprecations: [] }
     end
 
     # Loads a 'bolt.yaml' file, the legacy configuration file. There's no special munging needed
@@ -150,11 +156,12 @@ module Bolt
     def self.load_bolt_yaml(dir)
       filepath = dir + BOLT_CONFIG_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
+      logs     = [{ debug: "Loaded configuration from #{filepath}" }]
       deprecations = [{ type: 'Using bolt.yaml for system configuration',
                         msg: "Configuration file #{filepath} is deprecated and will be removed in a future version "\
                         "of Bolt. Use '#{dir + BOLT_DEFAULTS_NAME}' instead." }]
 
-      { filepath: filepath, data: data, warnings: [], deprecations: deprecations }
+      { filepath: filepath, data: data, logs: logs, deprecations: deprecations }
     end
 
     def self.load_defaults(project)
@@ -187,13 +194,13 @@ module Bolt
       unless config_data.is_a?(Array)
         config_data = [{ filepath: project.config_file,
                          data: config_data,
-                         warnings: [],
+                         logs: [],
                          deprecations: [] }]
       end
 
-      @logger       = Logging.logger[self]
+      @logger       = Bolt::Logger.logger(self)
       @project      = project
-      @warnings     = @project.warnings.dup
+      @logs         = @project.logs.dup
       @deprecations = @project.deprecations.dup
       @transports   = {}
       @config_files = []
@@ -221,7 +228,7 @@ module Bolt
       end
 
       loaded_data = config_data.each_with_object([]) do |data, acc|
-        @warnings.concat(data[:warnings]) if data[:warnings].any?
+        @logs.concat(data[:logs]) if data[:logs].any?
         @deprecations.concat(data[:deprecations]) if data[:deprecations].any?
 
         if data[:data].any?
@@ -337,10 +344,26 @@ module Bolt
     end
 
     private def update_logs(logs)
+      begin
+        if logs['bolt-debug.log'] && logs['bolt-debug.log'] != 'disable'
+          FileUtils.touch(File.expand_path('bolt-debug.log', @project.path))
+        end
+      rescue StandardError
+        logs.delete('bolt-debug.log')
+      end
+
       logs.each_with_object({}) do |(key, val), acc|
-        next unless val.is_a?(Hash)
+        # Remove any disabled logs
+        next if val == 'disable'
 
         name = normalize_log(key)
+
+        # But otherwise it has to be a Hash
+        unless val.is_a?(Hash)
+          raise Bolt::ValidationError,
+                "config of log #{name} must be a Hash, received #{val.class} #{val.inspect}"
+        end
+
         acc[name] = val.slice('append', 'level')
                        .transform_keys(&:to_sym)
 
@@ -365,7 +388,7 @@ module Bolt
     def validate
       if @data['future']
         msg = "Configuration option 'future' no longer exposes future behavior."
-        @warnings << { option: 'future', msg: msg }
+        @logs << { warn: msg }
       end
 
       keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]

data/lib/bolt/config/options.rb

@@ -176,7 +176,9 @@ module Bolt
           description: "A map of configuration for the logfile output. Under `log`, you can configure log options "\
                        "for `console` and add configuration for individual log files, such as "\
                        "`~/.puppetlabs/bolt/debug.log`. Individual log files must be valid filepaths. If the log "\
-                       "file does not exist, then Bolt will create it before logging information.",
+                       "file does not exist, then Bolt will create it before logging information. Set the value to "\
+                       "`disable` to remove a log file defined at an earlier level of the config hierarchy. By "\
+                       "default, Bolt logs to a bolt-debug.log file in the Bolt project directory.",
           type: Hash,
           properties: {
             "console" => {
@@ -194,7 +196,8 @@ module Bolt
           },
           additionalProperties: {
             description: "Configuration for the logfile output.",
-            type: Hash,
+            type: [String, Hash],
+            enum: ['disable'],
             properties: {
               "append" => {
                 description: "Whether to append output to an existing log file.",
@@ -224,6 +227,34 @@ module Bolt
           _example: ["~/.puppetlabs/bolt/modules", "~/.puppetlabs/bolt/site-modules"],
           _default: ["project/modules", "project/site-modules", "project/site"]
         },
+        "modules" => {
+          description: "A list of module dependencies for the project. Each dependency is a map of data specifying "\
+                       "the module to install. To install the project's module dependencies, run the `bolt module "\
+                       "install` command.",
+          type: Array,
+          items: {
+            type: Hash,
+            required: ["name"],
+            properties: {
+              "name" => {
+                description: "The name of the module.",
+                type: String
+              },
+              "version_requirement" => {
+                description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
+                             "shorthand (1.2.x), or a version range (>= 1.2.0).",
+                type: String
+              }
+            }
+          },
+          _plugin: false,
+          _example: [
+            { "name" => "puppetlabs-mysql" },
+            { "name" => "puppetlabs-apache", "version_requirement" => "5.5.0" },
+            { "name" => "puppetlabs-puppetdb", "version_requirement" => "7.x" },
+            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" }
+          ]
+        },
         "name" => {
           description: "The name of the Bolt project. When this option is configured, the project is considered a "\
                        "[Bolt project](experimental_features.md#bolt-projects), allowing Bolt to load content from "\
@@ -473,6 +504,7 @@ module Bolt
         inventoryfile
         log
         modulepath
+        modules
         name
         plans
         plugin_hooks

data/lib/bolt/executor.rb

@@ -40,7 +40,7 @@ module Bolt
       require 'concurrent'
 
       @analytics = analytics
-      @logger = Logging.logger[self]
+      @logger = Bolt::Logger.logger(self)
 
       @transports = Bolt::TRANSPORTS.each_with_object({}) do |(key, val), coll|
         coll[key.to_s] = if key == :remote

data/lib/bolt/inventory.rb

@@ -46,10 +46,13 @@ module Bolt
     end
 
     def self.from_config(config, plugins)
+      logger = Bolt::Logger.logger(self)
+
       if ENV.include?(ENVIRONMENT_VAR)
         begin
           data = YAML.safe_load(ENV[ENVIRONMENT_VAR])
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}" unless data.is_a?(Hash)
+          logger.debug("Loaded inventory from environment variable #{ENVIRONMENT_VAR}")
         rescue Psych::Exception
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
@@ -57,8 +60,12 @@ module Bolt
         data = if config.inventoryfile
                  Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
                else
-                 Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+                 i = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+                 logger.debug("Loaded inventory from #{config.default_inventoryfile}") if i
+                 i
                end
+        # This avoids rubocop complaining about identical conditionals
+        logger.debug("Loaded inventory from #{config.inventoryfile}") if config.inventoryfile
       end
 
       # Resolve plugin references from transport config

data/lib/bolt/inventory/group.rb

@@ -19,7 +19,7 @@ module Bolt
       CONFIG_KEYS = Bolt::Config::INVENTORY_OPTIONS.keys
 
       def initialize(input, plugins)
-        @logger = Logging.logger[self]
+        @logger = Bolt::Logger.logger(self)
         @plugins = plugins
 
         input = @plugins.resolve_top_level_references(input) if @plugins.reference?(input)

data/lib/bolt/inventory/inventory.rb

@@ -16,7 +16,7 @@ module Bolt
 
       # TODO: Pass transport config instead of config object
       def initialize(data, transport, transports, plugins)
-        @logger       = Logging.logger[self]
+        @logger       = Bolt::Logger.logger(self)
         @data         = data || {}
         @transport    = transport
         @config       = transports

data/lib/bolt/inventory/target.rb

@@ -13,7 +13,7 @@ module Bolt
           raise Bolt::Inventory::ValidationError.new("Target must have either a name or uri", nil)
         end
 
-        @logger = Logging.logger[inventory]
+        @logger = Bolt::Logger.logger(inventory)
 
         # If the target isn't mentioned by any groups, it won't have a uri or
         # name and we will use the target_name as both

data/lib/bolt/logger.rb

@@ -4,6 +4,10 @@ require 'logging'
 
 module Bolt
   module Logger
+    LEVELS = %w[trace debug info notice warn error fatal].freeze
+    @mutex = Mutex.new
+    @warnings = Set.new
+
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
     # levels which create corresponding methods on the logger instances;
@@ -11,24 +15,29 @@ module Bolt
     # will fail.
     def self.initialize_logging
       # Initialization isn't idempotent and will result in warnings about const
-      # redefs, so skip it if it's already been initialized
-      return if Logging.initialized?
-
-      Logging.init :trace, :debug, :info, :notice, :warn, :error, :fatal, :any
-      @mutex = Mutex.new
-
-      Logging.color_scheme(
-        'bolt',
-        lines: {
-          warn: :yellow,
-          error: :red,
-          fatal: %i[white on_red]
-        }
-      )
+      # redefs, so skip it if the log levels we expect are present. If it's
+      # already been initialized with an insufficient set of levels, go ahead
+      # and call init anyway or we'll have failures when calling log methods
+      # for missing levels.
+      unless levels & LEVELS == LEVELS
+        Logging.init(*LEVELS)
+      end
+
+      # As above, only create the color scheme if we haven't already created it.
+      unless Logging.color_scheme('bolt')
+        Logging.color_scheme(
+          'bolt',
+          lines: {
+            warn: :yellow,
+            error: :red,
+            fatal: %i[white on_red]
+          }
+        )
+      end
     end
 
     def self.configure(destinations, color)
-      root_logger = Logging.logger[:root]
+      root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
         'console',
@@ -66,6 +75,13 @@ module Bolt
       end
     end
 
+    # A helper to ensure the Logging library is always initialized with our
+    # custom log levels before retrieving a Logger instance.
+    def self.logger(name)
+      initialize_logging
+      Logging.logger[name]
+    end
+
     def self.analytics=(analytics)
       @analytics = analytics
     end
@@ -108,14 +124,12 @@ module Bolt
     end
 
     def self.warn_once(type, msg)
-      @mutex.synchronize {
-        @warnings ||= []
-        @logger ||= Logging.logger[self]
-        unless @warnings.include?(type)
+      @mutex.synchronize do
+        @logger ||= Bolt::Logger.logger(self)
+        if @warnings.add?(type)
           @logger.warn(msg)
-          @warnings << type
         end
-      }
+      end
     end
 
     def self.deprecation_warning(type, msg)

data/lib/bolt/outputter/logger.rb

@@ -7,7 +7,7 @@ module Bolt
     class Logger < Bolt::Outputter
       def initialize(verbose, trace)
         super(false, verbose, trace)
-        @logger = Logging.logger[self]
+        @logger = Bolt::Logger.logger(self)
       end
 
       def handle_event(event)

data/lib/bolt/pal.rb

@@ -65,7 +65,7 @@ module Bolt
       @resource_types = resource_types
       @project = project
 
-      @logger = Logging.logger[self]
+      @logger = Bolt::Logger.logger(self)
       if modulepath && !modulepath.empty?
         @logger.debug("Loading modules from #{@modulepath.join(File::PATH_SEPARATOR)}")
       end
@@ -76,7 +76,7 @@ module Bolt
     # Puppet logging is global so this is class method to avoid confusion
     def self.configure_logging
       Puppet::Util::Log.destinations.clear
-      Puppet::Util::Log.newdestination(Logging.logger['Puppet'])
+      Puppet::Util::Log.newdestination(Bolt::Logger.logger('Puppet'))
       # Defer all log level decisions to the Logging library by telling Puppet
       # to log everything
       Puppet.settings[:log_level] = 'debug'
@@ -141,6 +141,19 @@ module Bolt
       end
     end
 
+    def detect_project_conflict(project, environment)
+      return unless project && project.load_as_module?
+      # The environment modulepath has stripped out non-existent directories,
+      # so we don't need to check for them
+      modules = environment.modulepath.flat_map do |path|
+        Dir.children(path).select { |name| Puppet::Module.is_module_directory?(name, path) }
+      end
+      if modules.include?(project.name)
+        Bolt::Logger.warn_once("project shadows module",
+                               "The project '#{project.name}' shadows an existing module of the same name")
+      end
+    end
+
     # Runs a block in a PAL script compiler configured for Bolt.  Catches
     # exceptions thrown by the block and re-raises them ensuring they are
     # Bolt::Errors since the script compiler block will squash all exceptions.
@@ -149,15 +162,13 @@ module Bolt
       setup
       r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
-        bolt_project = @project if @project&.name
-        # Puppet currently won't receive the project unless it is a named project. Since
-        # the download_file plan function needs access to the project path, add it to the
-        # context.
-        bolt_project_data = @project
-        Puppet.override(bolt_project: bolt_project,
-                        bolt_project_data: bolt_project_data,
+        Puppet.override(bolt_project: @project,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
-          pal.with_script_compiler do |compiler|
+          # Because this has the side effect of loading and caching the list
+          # of modules, it must happen *after* we have overridden
+          # bolt_project or the project will be ignored
+          detect_project_conflict(@project, Puppet.lookup(:environments).get('bolt'))
+          pal.with_script_compiler(set_local_facts: false) do |compiler|
             alias_types(compiler)
             register_resource_types(Puppet.lookup(:loaders)) if @resource_types
             begin