bolt 2.25.0 → 2.30.0

data/lib/bolt/config.rb

@@ -19,7 +19,7 @@ module Bolt
   class Config
     include Bolt::Config::Options
 
-    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency, :deprecations
+    attr_reader :config_files, :logs, :data, :transports, :project, :modified_concurrency, :deprecations
 
     BOLT_CONFIG_NAME = 'bolt.yaml'
     BOLT_DEFAULTS_NAME = 'bolt-defaults.yaml'
@@ -32,16 +32,19 @@ module Bolt
     end
 
     def self.from_project(project, overrides = {})
+      logs = []
       conf = if project.project_file == project.config_file
                project.data
              else
-               Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+               c = Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+               logs << { debug: "Loaded configuration from #{project.config_file}" } if File.exist?(project.config_file)
+               c
              end
 
       data = load_defaults(project).push(
         filepath: project.config_file,
         data: conf,
-        warnings: [],
+        logs: logs,
         deprecations: []
       )
 
@@ -50,17 +53,20 @@ module Bolt
 
     def self.from_file(configfile, overrides = {})
       project = Bolt::Project.create_project(Pathname.new(configfile).expand_path.dirname)
+      logs = []
 
       conf = if project.project_file == project.config_file
                project.data
              else
-               Bolt::Util.read_yaml_hash(configfile, 'config')
+               c = Bolt::Util.read_yaml_hash(configfile, 'config')
+               logs << { debug: "Loaded configuration from #{configfile}" }
+               c
              end
 
       data = load_defaults(project).push(
-        filepath: project.config_file,
+        filepath: configfile,
         data: conf,
-        warnings: [],
+        logs: logs,
         deprecations: []
       )
 
@@ -90,13 +96,13 @@ module Bolt
     def self.load_bolt_defaults_yaml(dir)
       filepath = dir + BOLT_DEFAULTS_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
-      warnings = []
+      logs     = [{ debug: "Loaded configuration from #{filepath}" }]
 
       # Warn if 'bolt.yaml' detected in same directory.
       if File.exist?(bolt_yaml = dir + BOLT_CONFIG_NAME)
-        warnings.push(
-          msg: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
-               "will be ignored."
+        logs.push(
+          warn: "Detected multiple configuration files: ['#{bolt_yaml}', '#{filepath}']. '#{bolt_yaml}' "\
+          "will be ignored."
         )
       end
 
@@ -105,9 +111,9 @@ module Bolt
 
       if project_config.any?
         data.reject! { |key, _| project_config.include?(key) }
-        warnings.push(
-          msg: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
-                "Project configuration should be set in 'bolt-project.yaml'."
+        logs.push(
+          warn: "Unsupported project configuration detected in '#{filepath}': #{project_config.keys}. "\
+          "Project configuration should be set in 'bolt-project.yaml'."
         )
       end
 
@@ -116,10 +122,10 @@ module Bolt
 
       if transport_config.any?
         data.reject! { |key, _| transport_config.include?(key) }
-        warnings.push(
-          msg: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
-               "Transport configuration should be set under the 'inventory-config' option or "\
-               "in 'inventory.yaml'."
+        logs.push(
+          warn: "Unsupported inventory configuration detected in '#{filepath}': #{transport_config.keys}. "\
+          "Transport configuration should be set under the 'inventory-config' option or "\
+          "in 'inventory.yaml'."
         )
       end
 
@@ -142,7 +148,7 @@ module Bolt
         data = data.merge(data.delete('inventory-config'))
       end
 
-      { filepath: filepath, data: data, warnings: warnings, deprecations: [] }
+      { filepath: filepath, data: data, logs: logs, deprecations: [] }
     end
 
     # Loads a 'bolt.yaml' file, the legacy configuration file. There's no special munging needed
@@ -150,11 +156,12 @@ module Bolt
     def self.load_bolt_yaml(dir)
       filepath = dir + BOLT_CONFIG_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
+      logs     = [{ debug: "Loaded configuration from #{filepath}" }]
       deprecations = [{ type: 'Using bolt.yaml for system configuration',
                         msg: "Configuration file #{filepath} is deprecated and will be removed in a future version "\
                         "of Bolt. Use '#{dir + BOLT_DEFAULTS_NAME}' instead." }]
 
-      { filepath: filepath, data: data, warnings: [], deprecations: deprecations }
+      { filepath: filepath, data: data, logs: logs, deprecations: deprecations }
     end
 
     def self.load_defaults(project)
@@ -187,13 +194,13 @@ module Bolt
       unless config_data.is_a?(Array)
         config_data = [{ filepath: project.config_file,
                          data: config_data,
-                         warnings: [],
+                         logs: [],
                          deprecations: [] }]
       end
 
-      @logger       = Logging.logger[self]
+      @logger       = Bolt::Logger.logger(self)
       @project      = project
-      @warnings     = @project.warnings.dup
+      @logs         = @project.logs.dup
       @deprecations = @project.deprecations.dup
       @transports   = {}
       @config_files = []
@@ -221,7 +228,7 @@ module Bolt
       end
 
       loaded_data = config_data.each_with_object([]) do |data, acc|
-        @warnings.concat(data[:warnings]) if data[:warnings].any?
+        @logs.concat(data[:logs]) if data[:logs].any?
         @deprecations.concat(data[:deprecations]) if data[:deprecations].any?
 
         if data[:data].any?
@@ -337,10 +344,26 @@ module Bolt
     end
 
     private def update_logs(logs)
+      begin
+        if logs['bolt-debug.log'] && logs['bolt-debug.log'] != 'disable'
+          FileUtils.touch(File.expand_path('bolt-debug.log', @project.path))
+        end
+      rescue StandardError
+        logs.delete('bolt-debug.log')
+      end
+
       logs.each_with_object({}) do |(key, val), acc|
-        next unless val.is_a?(Hash)
+        # Remove any disabled logs
+        next if val == 'disable'
 
         name = normalize_log(key)
+
+        # But otherwise it has to be a Hash
+        unless val.is_a?(Hash)
+          raise Bolt::ValidationError,
+                "config of log #{name} must be a Hash, received #{val.class} #{val.inspect}"
+        end
+
         acc[name] = val.slice('append', 'level')
                        .transform_keys(&:to_sym)
 
@@ -365,7 +388,13 @@ module Bolt
     def validate
       if @data['future']
         msg = "Configuration option 'future' no longer exposes future behavior."
-        @warnings << { option: 'future', msg: msg }
+        @logs << { warn: msg }
+      end
+
+      if @project.modules && @data['modulepath']&.include?(@project.managed_moduledir.to_s)
+        raise Bolt::ValidationError,
+              "Found invalid path in modulepath: #{@project.managed_moduledir}. This path "\
+              "is automatically appended to the modulepath and cannot be configured."
       end
 
       keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
@@ -422,7 +451,13 @@ module Bolt
     end
 
     def modulepath
-      @data['modulepath'] || @project.modulepath
+      path = @data['modulepath'] || @project.modulepath
+
+      if @project.modules
+        path + [@project.managed_moduledir.to_s]
+      else
+        path
+      end
     end
 
     def modulepath=(value)

data/lib/bolt/config/options.rb

@@ -176,7 +176,9 @@ module Bolt
           description: "A map of configuration for the logfile output. Under `log`, you can configure log options "\
                        "for `console` and add configuration for individual log files, such as "\
                        "`~/.puppetlabs/bolt/debug.log`. Individual log files must be valid filepaths. If the log "\
-                       "file does not exist, then Bolt will create it before logging information.",
+                       "file does not exist, then Bolt will create it before logging information. Set the value to "\
+                       "`disable` to remove a log file defined at an earlier level of the config hierarchy. By "\
+                       "default, Bolt logs to a bolt-debug.log file in the Bolt project directory.",
           type: Hash,
           properties: {
             "console" => {
@@ -194,7 +196,8 @@ module Bolt
           },
           additionalProperties: {
             description: "Configuration for the logfile output.",
-            type: Hash,
+            type: [String, Hash],
+            enum: ['disable'],
             properties: {
               "append" => {
                 description: "Whether to append output to an existing log file.",
@@ -224,6 +227,35 @@ module Bolt
           _example: ["~/.puppetlabs/bolt/modules", "~/.puppetlabs/bolt/site-modules"],
           _default: ["project/modules", "project/site-modules", "project/site"]
         },
+        "modules" => {
+          description: "A list of module dependencies for the project. Each dependency is a map of data specifying "\
+                       "the module to install. To install the project's module dependencies, run the `bolt module "\
+                       "install` command.",
+          type: Array,
+          items: {
+            type: [Hash, String],
+            required: ["name"],
+            properties: {
+              "name" => {
+                description: "The name of the module.",
+                type: String
+              },
+              "version_requirement" => {
+                description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
+                             "shorthand (1.2.x), or a version range (>= 1.2.0).",
+                type: String
+              }
+            }
+          },
+          _plugin: false,
+          _example: [
+            { "name" => "puppetlabs-mysql" },
+            "puppetlabs-facts",
+            { "name" => "puppetlabs-apache", "version_requirement" => "5.5.0" },
+            { "name" => "puppetlabs-puppetdb", "version_requirement" => "7.x" },
+            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" }
+          ]
+        },
         "name" => {
           description: "The name of the Bolt project. When this option is configured, the project is considered a "\
                        "[Bolt project](experimental_features.md#bolt-projects), allowing Bolt to load content from "\
@@ -473,6 +505,7 @@ module Bolt
         inventoryfile
         log
         modulepath
+        modules
         name
         plans
         plugin_hooks

data/lib/bolt/executor.rb

@@ -40,7 +40,7 @@ module Bolt
       require 'concurrent'
 
       @analytics = analytics
-      @logger = Logging.logger[self]
+      @logger = Bolt::Logger.logger(self)
 
       @transports = Bolt::TRANSPORTS.each_with_object({}) do |(key, val), coll|
         coll[key.to_s] = if key == :remote
@@ -227,7 +227,7 @@ module Bolt
         data[:resource_mean] = sum / resource_counts.length
       end
 
-      @analytics&.event('Apply', 'ast', data)
+      @analytics&.event('Apply', 'ast', **data)
     end
 
     def report_yaml_plan(plan)

data/lib/bolt/inventory.rb

@@ -46,10 +46,13 @@ module Bolt
     end
 
     def self.from_config(config, plugins)
+      logger = Bolt::Logger.logger(self)
+
       if ENV.include?(ENVIRONMENT_VAR)
         begin
           data = YAML.safe_load(ENV[ENVIRONMENT_VAR])
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}" unless data.is_a?(Hash)
+          logger.debug("Loaded inventory from environment variable #{ENVIRONMENT_VAR}")
         rescue Psych::Exception
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
@@ -57,8 +60,12 @@ module Bolt
         data = if config.inventoryfile
                  Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
                else
-                 Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+                 i = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+                 logger.debug("Loaded inventory from #{config.default_inventoryfile}") if i
+                 i
                end
+        # This avoids rubocop complaining about identical conditionals
+        logger.debug("Loaded inventory from #{config.inventoryfile}") if config.inventoryfile
       end
 
       # Resolve plugin references from transport config

data/lib/bolt/inventory/group.rb

@@ -19,7 +19,7 @@ module Bolt
       CONFIG_KEYS = Bolt::Config::INVENTORY_OPTIONS.keys
 
       def initialize(input, plugins)
-        @logger = Logging.logger[self]
+        @logger = Bolt::Logger.logger(self)
         @plugins = plugins
 
         input = @plugins.resolve_top_level_references(input) if @plugins.reference?(input)

data/lib/bolt/inventory/inventory.rb

@@ -16,7 +16,7 @@ module Bolt
 
       # TODO: Pass transport config instead of config object
       def initialize(data, transport, transports, plugins)
-        @logger       = Logging.logger[self]
+        @logger       = Bolt::Logger.logger(self)
         @data         = data || {}
         @transport    = transport
         @config       = transports

data/lib/bolt/inventory/target.rb

@@ -13,7 +13,7 @@ module Bolt
           raise Bolt::Inventory::ValidationError.new("Target must have either a name or uri", nil)
         end
 
-        @logger = Logging.logger[inventory]
+        @logger = Bolt::Logger.logger(inventory)
 
         # If the target isn't mentioned by any groups, it won't have a uri or
         # name and we will use the target_name as both

data/lib/bolt/logger.rb

@@ -4,6 +4,10 @@ require 'logging'
 
 module Bolt
   module Logger
+    LEVELS = %w[trace debug info notice warn error fatal].freeze
+    @mutex = Mutex.new
+    @warnings = Set.new
+
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
     # levels which create corresponding methods on the logger instances;
@@ -11,24 +15,29 @@ module Bolt
     # will fail.
     def self.initialize_logging
       # Initialization isn't idempotent and will result in warnings about const
-      # redefs, so skip it if it's already been initialized
-      return if Logging.initialized?
-
-      Logging.init :trace, :debug, :info, :notice, :warn, :error, :fatal, :any
-      @mutex = Mutex.new
-
-      Logging.color_scheme(
-        'bolt',
-        lines: {
-          warn: :yellow,
-          error: :red,
-          fatal: %i[white on_red]
-        }
-      )
+      # redefs, so skip it if the log levels we expect are present. If it's
+      # already been initialized with an insufficient set of levels, go ahead
+      # and call init anyway or we'll have failures when calling log methods
+      # for missing levels.
+      unless levels & LEVELS == LEVELS
+        Logging.init(*LEVELS)
+      end
+
+      # As above, only create the color scheme if we haven't already created it.
+      unless Logging.color_scheme('bolt')
+        Logging.color_scheme(
+          'bolt',
+          lines: {
+            warn: :yellow,
+            error: :red,
+            fatal: %i[white on_red]
+          }
+        )
+      end
     end
 
     def self.configure(destinations, color)
-      root_logger = Logging.logger[:root]
+      root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
         'console',
@@ -66,6 +75,13 @@ module Bolt
       end
     end
 
+    # A helper to ensure the Logging library is always initialized with our
+    # custom log levels before retrieving a Logger instance.
+    def self.logger(name)
+      initialize_logging
+      Logging.logger[name]
+    end
+
     def self.analytics=(analytics)
       @analytics = analytics
     end
@@ -108,14 +124,12 @@ module Bolt
     end
 
     def self.warn_once(type, msg)
-      @mutex.synchronize {
-        @warnings ||= []
-        @logger ||= Logging.logger[self]
-        unless @warnings.include?(type)
+      @mutex.synchronize do
+        @logger ||= Bolt::Logger.logger(self)
+        if @warnings.add?(type)
           @logger.warn(msg)
-          @warnings << type
         end
-      }
+      end
     end
 
     def self.deprecation_warning(type, msg)

data/lib/bolt/module_installer.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require 'bolt/error'
+require 'bolt/logger'
+
+module Bolt
+  class ModuleInstaller
+    def initialize(outputter, pal)
+      @outputter = outputter
+      @pal       = pal
+      @logger    = Bolt::Logger.logger(self)
+    end
+
+    # Adds a single module to the project.
+    #
+    def add(name, modules, puppetfile_path, moduledir, config_path)
+      require 'bolt/puppetfile'
+
+      # If the project configuration file already includes this module,
+      # exit early.
+      puppetfile  = Bolt::Puppetfile.new(modules)
+      new_module  = Bolt::Puppetfile::Module.from_hash('name' => name)
+
+      if puppetfile.modules.include?(new_module)
+        @outputter.print_message "Project configuration file #{config_path} already "\
+                                 "includes module #{new_module}. Nothing to do."
+        return true
+      end
+
+      # If the Puppetfile exists, make sure it's managed by Bolt.
+      if puppetfile_path.exist?
+        assert_managed_puppetfile(puppetfile, puppetfile_path)
+      end
+
+      # Create a Puppetfile object that includes the new module and its
+      # dependencies. We error early here so we don't add the new module to the
+      # project config or modify the Puppetfile.
+      puppetfile = add_new_module_to_puppetfile(new_module, modules, puppetfile_path)
+
+      # Add the module to the project configuration.
+      @outputter.print_message "Updating project configuration file at #{config_path}"
+
+      data = Bolt::Util.read_yaml_hash(config_path, 'project')
+      data['modules'] ||= []
+      data['modules'] <<  { 'name' => new_module.title }
+
+      begin
+        File.write(config_path, data.to_yaml)
+      rescue SystemCallError => e
+        raise Bolt::FileError.new(
+          "Unable to update project configuration file: #{e.message}",
+          config
+        )
+      end
+
+      # Write the Puppetfile.
+      @outputter.print_message "Writing Puppetfile at #{puppetfile_path}"
+      puppetfile.write(puppetfile_path, moduledir)
+
+      # Install the modules.
+      install_puppetfile(puppetfile_path, moduledir)
+    end
+
+    # Creates a new Puppetfile that includes the new module and its dependencies.
+    #
+    private def add_new_module_to_puppetfile(new_module, modules, path)
+      @outputter.print_message "Resolving module dependencies, this may take a moment"
+
+      # If there is an existing Puppetfile, add the new module and attempt
+      # to resolve. This will not update the versions of any installed modules.
+      if path.exist?
+        puppetfile = Bolt::Puppetfile.parse(path)
+        puppetfile.add_modules(new_module)
+
+        begin
+          puppetfile.resolve
+          return puppetfile
+        rescue Bolt::Error
+          @logger.debug "Unable to find a version of #{new_module} compatible "\
+                        "with installed modules. Attempting to re-resolve modules "\
+                        "from project configuration; some versions of installed "\
+                        "modules may change."
+        end
+      end
+
+      # If there is not an existing Puppetfile, or resolving with pinned
+      # modules fails, resolve all of the module declarations with the new
+      # module.
+      puppetfile = Bolt::Puppetfile.new(modules)
+      puppetfile.add_modules(new_module)
+      puppetfile.resolve
+      puppetfile
+    end
+
+    # Installs a project's module dependencies.
+    #
+    def install(modules, path, moduledir, force: false, resolve: true)
+      require 'bolt/puppetfile'
+
+      puppetfile = Bolt::Puppetfile.new(modules)
+
+      # If the Puppetfile exists, check if it includes specs for each declared
+      # module, erroring if there are any missing. Otherwise, resolve the
+      # module dependencies and write a new Puppetfile. Users can forcibly
+      # overwrite an existing Puppetfile with the '--force' option, or opt to
+      # install the Puppetfile as-is with --no-resolve.
+      #
+      # This is just if resolve is not false (nil should default to true)
+      if resolve != false
+        if path.exist? && !force
+          assert_managed_puppetfile(puppetfile, path)
+        else
+          @outputter.print_message "Resolving module dependencies, this may take a moment"
+          puppetfile.resolve
+
+          @outputter.print_message "Writing Puppetfile at #{path}"
+          # We get here either through 'bolt module install' which uses the
+          # managed modulepath (which isn't configurable) or through bolt
+          # project init --modules, which uses the default modulepath. This
+          # should be safe to assume that if `.modules/` is the moduledir the
+          # user is using the new workflow
+          if moduledir.basename == '.modules'
+            puppetfile.write(path, moduledir)
+          else
+            puppetfile.write(path)
+          end
+        end
+      end
+
+      # Install the modules.
+      install_puppetfile(path, moduledir)
+    end
+
+    # Installs the Puppetfile and generates types.
+    #
+    def install_puppetfile(path, moduledir, config = {})
+      require 'bolt/puppetfile/installer'
+
+      @outputter.print_message "Syncing modules from #{path} to #{moduledir}"
+      ok = Bolt::Puppetfile::Installer.new(config).install(path, moduledir)
+
+      # Automatically generate types after installing modules
+      @pal.generate_types
+
+      @outputter.print_puppetfile_result(ok, path, moduledir)
+
+      ok
+    end
+
+    # Asserts that an existing Puppetfile is managed by Bolt.
+    #
+    private def assert_managed_puppetfile(puppetfile, path)
+      existing_puppetfile = Bolt::Puppetfile.parse(path)
+
+      unless existing_puppetfile.modules.superset? puppetfile.modules
+        missing_modules = puppetfile.modules - existing_puppetfile.modules
+
+        message = <<~MESSAGE.chomp
+          Puppetfile #{path} is missing specifications for the following
+          module declarations:
+
+          #{missing_modules.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
+          
+          This may not be a Puppetfile managed by Bolt. To forcibly overwrite the
+          Puppetfile, run 'bolt module install --force'.
+        MESSAGE
+
+        raise Bolt::Error.new(message, 'bolt/missing-module-specs')
+      end
+    end
+  end
+end