bolt 2.26.0 → 2.31.0

data/lib/bolt/config.rb

@@ -64,7 +64,7 @@ module Bolt
              end
 
       data = load_defaults(project).push(
-        filepath: project.config_file,
+        filepath: configfile,
         data: conf,
         logs: logs,
         deprecations: []
@@ -344,6 +344,14 @@ module Bolt
     end
 
     private def update_logs(logs)
+      begin
+        if logs['bolt-debug.log'] && logs['bolt-debug.log'] != 'disable'
+          FileUtils.touch(File.expand_path('bolt-debug.log', @project.path))
+        end
+      rescue StandardError
+        logs.delete('bolt-debug.log')
+      end
+
       logs.each_with_object({}) do |(key, val), acc|
         # Remove any disabled logs
         next if val == 'disable'
@@ -383,6 +391,12 @@ module Bolt
         @logs << { warn: msg }
       end
 
+      if @project.modules && @data['modulepath']&.include?(@project.managed_moduledir.to_s)
+        raise Bolt::ValidationError,
+              "Found invalid path in modulepath: #{@project.managed_moduledir}. This path "\
+              "is automatically appended to the modulepath and cannot be configured."
+      end
+
       keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
@@ -437,7 +451,13 @@ module Bolt
     end
 
     def modulepath
-      @data['modulepath'] || @project.modulepath
+      path = @data['modulepath'] || @project.modulepath
+
+      if @project.modules
+        path + [@project.managed_moduledir.to_s]
+      else
+        path
+      end
     end
 
     def modulepath=(value)

data/lib/bolt/config/modulepath.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'bolt/config'
+
+module Bolt
+  class Config
+    class Modulepath
+      BOLTLIB_PATH = File.expand_path('../../../bolt-modules', __dir__)
+      MODULES_PATH = File.expand_path('../../../modules', __dir__)
+
+      # The user_modulepath only includes the original modulepath and is used during pluginsync.
+      # We don't want to pluginsync any of the content from BOLT_MODULES since that content
+      # includes core modules that can conflict with modules installed with an agent.
+      attr_reader :user_modulepath
+
+      def initialize(user_modulepath, boltlib_path: BOLTLIB_PATH, builtin_content_path: MODULES_PATH)
+        @user_modulepath = Array(user_modulepath).flatten
+        @boltlib_path = Array(boltlib_path).flatten
+        @builtin_content_path = Array(builtin_content_path).flatten
+      end
+
+      # The full_modulepath includes both the BOLTLIB
+      # path and the MODULES_PATH to ensure bolt functions and
+      # built-in content are available in the compliler
+      def full_modulepath
+        @boltlib_path + @user_modulepath + @builtin_content_path
+      end
+    end
+  end
+end

data/lib/bolt/config/options.rb

@@ -227,6 +227,35 @@ module Bolt
           _example: ["~/.puppetlabs/bolt/modules", "~/.puppetlabs/bolt/site-modules"],
           _default: ["project/modules", "project/site-modules", "project/site"]
         },
+        "modules" => {
+          description: "A list of module dependencies for the project. Each dependency is a map of data specifying "\
+                       "the module to install. To install the project's module dependencies, run the `bolt module "\
+                       "install` command.",
+          type: Array,
+          items: {
+            type: [Hash, String],
+            required: ["name"],
+            properties: {
+              "name" => {
+                description: "The name of the module.",
+                type: String
+              },
+              "version_requirement" => {
+                description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
+                             "shorthand (1.2.x), or a version range (>= 1.2.0).",
+                type: String
+              }
+            }
+          },
+          _plugin: false,
+          _example: [
+            { "name" => "puppetlabs-mysql" },
+            "puppetlabs-facts",
+            { "name" => "puppetlabs-apache", "version_requirement" => "5.5.0" },
+            { "name" => "puppetlabs-puppetdb", "version_requirement" => "7.x" },
+            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" }
+          ]
+        },
         "name" => {
           description: "The name of the Bolt project. When this option is configured, the project is considered a "\
                        "[Bolt project](experimental_features.md#bolt-projects), allowing Bolt to load content from "\
@@ -476,6 +505,7 @@ module Bolt
         inventoryfile
         log
         modulepath
+        modules
         name
         plans
         plugin_hooks

data/lib/bolt/config/transport/options.rb

@@ -374,7 +374,7 @@ module Bolt
           },
           "ssh-command" => {
             type: [Array, String],
-            description: "The command and flags to use when SSHing. This option is used when you need support for "\
+            description: "The command and options to use when SSHing. This option is used when you need support for "\
                          "features or algorithms that are not supported by the net-ssh Ruby library. **This option "\
                          "is experimental.** You can read more about this  option in [Native SSH "\
                          "transport](experimental_features.md#native-ssh-transport).",

data/lib/bolt/executor.rb

@@ -227,7 +227,7 @@ module Bolt
         data[:resource_mean] = sum / resource_counts.length
       end
 
-      @analytics&.event('Apply', 'ast', data)
+      @analytics&.event('Apply', 'ast', **data)
     end
 
     def report_yaml_plan(plan)

data/lib/bolt/inventory.rb

@@ -46,7 +46,7 @@ module Bolt
     end
 
     def self.from_config(config, plugins)
-      logger = Logging.logger[self]
+      logger = Bolt::Logger.logger(self)
 
       if ENV.include?(ENVIRONMENT_VAR)
         begin
@@ -56,16 +56,17 @@ module Bolt
         rescue Psych::Exception
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
+      elsif config.inventoryfile
+        data = Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
+        logger.debug("Loaded inventory from #{config.inventoryfile}")
       else
-        data = if config.inventoryfile
-                 Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
-               else
-                 i = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
-                 logger.debug("Loaded inventory from #{config.default_inventoryfile}") if i
-                 i
-               end
-        # This avoids rubocop complaining about identical conditionals
-        logger.debug("Loaded inventory from #{config.inventoryfile}") if config.inventoryfile
+        data = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+
+        if config.default_inventoryfile.exist?
+          logger.debug("Loaded inventory from #{config.default_inventoryfile}")
+        else
+          logger.debug("Tried to load inventory from #{config.default_inventoryfile}, but the file does not exist")
+        end
       end
 
       # Resolve plugin references from transport config

data/lib/bolt/logger.rb

@@ -4,6 +4,10 @@ require 'logging'
 
 module Bolt
   module Logger
+    LEVELS = %w[trace debug info notice warn error fatal].freeze
+    @mutex = Mutex.new
+    @warnings = Set.new
+
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
     # levels which create corresponding methods on the logger instances;
@@ -11,20 +15,25 @@ module Bolt
     # will fail.
     def self.initialize_logging
       # Initialization isn't idempotent and will result in warnings about const
-      # redefs, so skip it if it's already been initialized
-      return if Logging.initialized?
-
-      Logging.init :trace, :debug, :info, :notice, :warn, :error, :fatal, :any
-      @mutex = Mutex.new
-
-      Logging.color_scheme(
-        'bolt',
-        lines: {
-          warn: :yellow,
-          error: :red,
-          fatal: %i[white on_red]
-        }
-      )
+      # redefs, so skip it if the log levels we expect are present. If it's
+      # already been initialized with an insufficient set of levels, go ahead
+      # and call init anyway or we'll have failures when calling log methods
+      # for missing levels.
+      unless levels & LEVELS == LEVELS
+        Logging.init(*LEVELS)
+      end
+
+      # As above, only create the color scheme if we haven't already created it.
+      unless Logging.color_scheme('bolt')
+        Logging.color_scheme(
+          'bolt',
+          lines: {
+            warn: :yellow,
+            error: :red,
+            fatal: %i[white on_red]
+          }
+        )
+      end
     end
 
     def self.configure(destinations, color)
@@ -115,14 +124,12 @@ module Bolt
     end
 
     def self.warn_once(type, msg)
-      @mutex.synchronize {
-        @warnings ||= []
+      @mutex.synchronize do
         @logger ||= Bolt::Logger.logger(self)
-        unless @warnings.include?(type)
+        if @warnings.add?(type)
           @logger.warn(msg)
-          @warnings << type
         end
-      }
+      end
     end
 
     def self.deprecation_warning(type, msg)

data/lib/bolt/module_installer.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require 'bolt/error'
+require 'bolt/logger'
+
+module Bolt
+  class ModuleInstaller
+    def initialize(outputter, pal)
+      @outputter = outputter
+      @pal       = pal
+      @logger    = Bolt::Logger.logger(self)
+    end
+
+    # Adds a single module to the project.
+    #
+    def add(name, modules, puppetfile_path, moduledir, config_path)
+      require 'bolt/puppetfile'
+
+      @outputter.print_message("Adding module #{name} to project\n\n")
+
+      # If the project configuration file already includes this module,
+      # exit early.
+      puppetfile  = Bolt::Puppetfile.new(modules)
+      new_module  = Bolt::Puppetfile::Module.from_hash('name' => name)
+
+      if puppetfile.modules.include?(new_module)
+        @outputter.print_action_step(
+          "Project configuration file #{config_path} already includes module #{new_module}. Nothing to do."
+        )
+        return true
+      end
+
+      # If the Puppetfile exists, make sure it's managed by Bolt.
+      if puppetfile_path.exist?
+        assert_managed_puppetfile(puppetfile, puppetfile_path)
+        existing = Bolt::Puppetfile.parse(puppetfile_path)
+      else
+        existing = Bolt::Puppetfile.new
+      end
+
+      # Create a Puppetfile object that includes the new module and its
+      # dependencies. We error early here so we don't add the new module to the
+      # project config or modify the Puppetfile.
+      puppetfile = add_new_module_to_puppetfile(new_module, modules, puppetfile_path)
+
+      # Display the diff between the existing Puppetfile and the new Puppetfile.
+      print_puppetfile_diff(existing, puppetfile)
+
+      # Add the module to the project configuration.
+      @outputter.print_action_step("Updating project configuration file at #{config_path}")
+
+      data = Bolt::Util.read_yaml_hash(config_path, 'project')
+      data['modules'] ||= []
+      data['modules'] <<  { 'name' => new_module.title }
+
+      begin
+        File.write(config_path, data.to_yaml)
+      rescue SystemCallError => e
+        raise Bolt::FileError.new(
+          "Unable to update project configuration file: #{e.message}",
+          config
+        )
+      end
+
+      # Write the Puppetfile.
+      @outputter.print_action_step("Writing Puppetfile at #{puppetfile_path}")
+      puppetfile.write(puppetfile_path, moduledir)
+
+      # Install the modules.
+      install_puppetfile(puppetfile_path, moduledir)
+    end
+
+    # Creates a new Puppetfile that includes the new module and its dependencies.
+    #
+    private def add_new_module_to_puppetfile(new_module, modules, path)
+      @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+
+      # If there is an existing Puppetfile, add the new module and attempt
+      # to resolve. This will not update the versions of any installed modules.
+      if path.exist?
+        puppetfile = Bolt::Puppetfile.parse(path)
+        puppetfile.add_modules(new_module)
+
+        begin
+          puppetfile.resolve
+          return puppetfile
+        rescue Bolt::Error
+          @logger.debug "Unable to find a version of #{new_module} compatible "\
+                        "with installed modules. Attempting to re-resolve modules "\
+                        "from project configuration; some versions of installed "\
+                        "modules may change."
+        end
+      end
+
+      # If there is not an existing Puppetfile, or resolving with pinned
+      # modules fails, resolve all of the module declarations with the new
+      # module.
+      puppetfile = Bolt::Puppetfile.new(modules)
+      puppetfile.add_modules(new_module)
+      puppetfile.resolve
+      puppetfile
+    end
+
+    # Outputs a diff of an old Puppetfile and a new Puppetfile.
+    #
+    def print_puppetfile_diff(old, new)
+      # Build hashes mapping the module title to the module object. This makes it
+      # a little easier to determine which modules have been added, removed, or
+      # modified.
+      old = old.modules.each_with_object({}) do |mod, acc|
+        acc[mod.title] = mod
+      end
+
+      new = new.modules.each_with_object({}) do |mod, acc|
+        acc[mod.title] = mod
+      end
+
+      # New modules are those present in new but not in old.
+      added = new.reject { |title, _mod| old.include?(title) }.values
+
+      if added.any?
+        diff = "Adding the following modules:\n"
+        added.each { |mod| diff += "#{mod.title} #{mod.version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Upgraded modules are those that have a newer version in new than old.
+      upgraded = new.select do |title, mod|
+        if old.include?(title)
+          SemanticPuppet::Version.parse(mod.version) > SemanticPuppet::Version.parse(old[title].version)
+        end
+      end.keys
+
+      if upgraded.any?
+        diff = "Upgrading the following modules:\n"
+        upgraded.each { |title| diff += "#{title} #{old[title].version} to #{new[title].version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Downgraded modules are those that have an older version in new than old.
+      downgraded = new.select do |title, mod|
+        if old.include?(title)
+          SemanticPuppet::Version.parse(mod.version) < SemanticPuppet::Version.parse(old[title].version)
+        end
+      end.keys
+
+      if downgraded.any?
+        diff = "Downgrading the following modules: \n"
+        downgraded.each { |title| diff += "#{title} #{old[title].version} to #{new[title].version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Removed modules are those present in old but not in new.
+      removed = old.reject { |title, _mod| new.include?(title) }.values
+
+      if removed.any?
+        diff = "Removing the following modules:\n"
+        removed.each { |mod| diff += "#{mod.title} #{mod.version}\n" }
+        @outputter.print_action_step(diff)
+      end
+    end
+
+    # Installs a project's module dependencies.
+    #
+    def install(modules, path, moduledir, force: false, resolve: true)
+      require 'bolt/puppetfile'
+
+      @outputter.print_message("Installing project modules\n\n")
+
+      puppetfile = Bolt::Puppetfile.new(modules)
+
+      # If the Puppetfile exists, check if it includes specs for each declared
+      # module, erroring if there are any missing. Otherwise, resolve the
+      # module dependencies and write a new Puppetfile. Users can forcibly
+      # overwrite an existing Puppetfile with the '--force' option, or opt to
+      # install the Puppetfile as-is with --no-resolve.
+      #
+      # This is just if resolve is not false (nil should default to true)
+      if resolve != false
+        if path.exist? && !force
+          assert_managed_puppetfile(puppetfile, path)
+        else
+          @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+          puppetfile.resolve
+
+          @outputter.print_action_step("Writing Puppetfile at #{path}")
+          # We get here either through 'bolt module install' which uses the
+          # managed modulepath (which isn't configurable) or through bolt
+          # project init --modules, which uses the default modulepath. This
+          # should be safe to assume that if `.modules/` is the moduledir the
+          # user is using the new workflow
+          if moduledir.basename.to_s == '.modules'
+            puppetfile.write(path, moduledir)
+          else
+            puppetfile.write(path)
+          end
+        end
+      end
+
+      # Install the modules.
+      install_puppetfile(path, moduledir)
+    end
+
+    # Installs the Puppetfile and generates types.
+    #
+    def install_puppetfile(path, moduledir, config = {})
+      require 'bolt/puppetfile/installer'
+
+      @outputter.print_action_step("Syncing modules from #{path} to #{moduledir}")
+      ok = Bolt::Puppetfile::Installer.new(config).install(path, moduledir)
+
+      # Automatically generate types after installing modules
+      @pal.generate_types
+
+      @outputter.print_puppetfile_result(ok, path, moduledir)
+
+      ok
+    end
+
+    # Asserts that an existing Puppetfile is managed by Bolt.
+    #
+    private def assert_managed_puppetfile(puppetfile, path)
+      existing_puppetfile = Bolt::Puppetfile.parse(path)
+
+      unless existing_puppetfile.modules.superset? puppetfile.modules
+        missing_modules = puppetfile.modules - existing_puppetfile.modules
+
+        message = <<~MESSAGE.chomp
+          Puppetfile #{path} is missing specifications for the following
+          module declarations:
+
+          #{missing_modules.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
+          
+          This may not be a Puppetfile managed by Bolt. To forcibly overwrite the
+          Puppetfile, run 'bolt module install --force'.
+        MESSAGE
+
+        raise Bolt::Error.new(message, 'bolt/missing-module-specs')
+      end
+    end
+  end
+end