bolt 2.26.0 → 2.31.0

data/lib/bolt/outputter.rb

@@ -35,6 +35,10 @@ module Bolt
       raise NotImplementedError, "print_message() must be implemented by the outputter class"
     end
 
+    def print_error
+      raise NotImplementedError, "print_error() must be implemented by the outputter class"
+    end
+
     def stringify(message)
       formatted = format_message(message)
       if formatted.is_a?(Hash) || formatted.is_a?(Array)

data/lib/bolt/outputter/human.rb

@@ -5,9 +5,12 @@ require 'bolt/pal'
 module Bolt
   class Outputter
     class Human < Bolt::Outputter
-      COLORS = { red: "31",
-                 green: "32",
-                 yellow: "33" }.freeze
+      COLORS = {
+        red:    "31",
+        green:  "32",
+        yellow: "33",
+        cyan:   "36"
+      }.freeze
 
       def print_head; end
 
@@ -31,6 +34,10 @@ module Bolt
         string.sub(/\s\z/, '')
       end
 
+      def wrap(string, width = 80)
+        string.gsub(/(.{1,#{width}})(\s+|\Z)/, "\\1\n")
+      end
+
       def handle_event(event)
         case event[:type]
         when :enable_default_output
@@ -48,9 +55,9 @@ module Bolt
           when :node_result
             print_result(event[:result]) if @verbose
           when :step_start
-            print_step_start(event) if plan_logging?
+            print_step_start(**event) if plan_logging?
           when :step_finish
-            print_step_finish(event) if plan_logging?
+            print_step_finish(**event) if plan_logging?
           when :plan_start
             print_plan_start(event)
           when :plan_finish
@@ -188,7 +195,7 @@ module Bolt
         @stream.puts total_msg
       end
 
-      def print_table(results)
+      def print_table(results, padding_left = 0, padding_right = 3)
         # lazy-load expensive gem code
         require 'terminal-table'
 
@@ -198,8 +205,8 @@ module Bolt
             border_x: '',
             border_y: '',
             border_i: '',
-            padding_left: 0,
-            padding_right: 3,
+            padding_left: padding_left,
+            padding_right: padding_right,
             border_top: false,
             border_bottom: false
           }
@@ -241,7 +248,7 @@ module Bolt
         task_info << "MODULE:\n"
 
         path = task.files.first['path'].chomp("/tasks/#{task.files.first['name']}")
-        task_info << if path.start_with?(Bolt::PAL::MODULES_PATH)
+        task_info << if path.start_with?(Bolt::Config::Modulepath::MODULES_PATH)
                        "built-in module"
                      else
                        path
@@ -271,7 +278,7 @@ module Bolt
         plan_info << "MODULE:\n"
 
         path = plan['module']
-        plan_info << if path.start_with?(Bolt::PAL::MODULES_PATH)
+        plan_info << if path.start_with?(Bolt::Config::Modulepath::MODULES_PATH)
                        "built-in module"
                      else
                        path
@@ -299,9 +306,9 @@ module Bolt
       def print_module_list(module_list)
         module_list.each do |path, modules|
           if (mod = modules.find { |m| m[:internal_module_group] })
-            @stream.puts(mod[:internal_module_group])
+            @stream.puts(colorize(:cyan, mod[:internal_module_group]))
           else
-            @stream.puts(path)
+            @stream.puts(colorize(:cyan, path))
           end
 
           if modules.empty?
@@ -317,17 +324,35 @@ module Bolt
               [m[:name], version]
             end
 
-            print_table(module_info)
+            print_table(module_info, 2, 1)
           end
 
           @stream.write("\n")
         end
       end
 
-      def print_targets(targets)
-        count = "#{targets.count} target#{'s' unless targets.count == 1}"
-        @stream.puts targets.map(&:name).join("\n")
-        @stream.puts colorize(:green, count)
+      def print_targets(target_list, inventoryfile)
+        adhoc = colorize(:yellow, "(Not found in inventory file)")
+
+        targets  = []
+        targets += target_list[:inventory].map { |target| [target.name, nil] }
+        targets += target_list[:adhoc].map { |target| [target.name, adhoc] }
+
+        if targets.any?
+          print_table(targets, 0, 2)
+          @stream.puts
+        end
+
+        @stream.puts "INVENTORY FILE:"
+        if inventoryfile.exist?
+          @stream.puts inventoryfile
+        else
+          @stream.puts wrap("Tried to load inventory from #{inventoryfile}, but the file does not exist")
+        end
+
+        @stream.puts "\nTARGET COUNT:"
+        @stream.puts "#{targets.count} total, #{target_list[:inventory].count} from inventory, "\
+                     "#{target_list[:adhoc].count} adhoc"
       end
 
       def print_target_info(targets)
@@ -394,6 +419,41 @@ module Bolt
         @stream.puts(message)
       end
 
+      def print_error(message)
+        @stream.puts(colorize(:red, message))
+      end
+
+      def print_prompt(prompt)
+        @stream.print(colorize(:cyan, indent(4, prompt)))
+      end
+
+      def print_prompt_error(message)
+        @stream.puts(colorize(:red, indent(4, message)))
+      end
+
+      def print_action_step(step)
+        first, *remaining = wrap(step, 76).lines
+
+        first     = indent(2, "→ #{first}")
+        remaining = remaining.map { |line| indent(4, line) }
+        step      = [first, *remaining, "\n"].join
+
+        @stream.puts(step)
+      end
+
+      def print_action_error(error)
+        # Running everything through 'wrap' messes with newlines. Separating
+        # into lines and wrapping each individually ensures separate errors are
+        # distinguishable.
+        first, *remaining = error.lines
+        first = colorize(:red, indent(2, "→ #{wrap(first, 76)}"))
+        wrapped = remaining.map { |l| wrap(l) }
+        to_print = wrapped.map { |line| colorize(:red, indent(4, line)) }
+        step = [first, *to_print, "\n"].join
+
+        @stream.puts(step)
+      end
+
       def duration_to_string(duration)
         hrs = (duration / 3600).floor
         mins = ((duration % 3600) / 60).floor

data/lib/bolt/outputter/json.rb

@@ -48,7 +48,7 @@ module Bolt
 
       def print_task_info(task)
         path = task.files.first['path'].chomp("/tasks/#{task.files.first['name']}")
-        module_dir = if path.start_with?(Bolt::PAL::MODULES_PATH)
+        module_dir = if path.start_with?(Bolt::Config::Modulepath::MODULES_PATH)
                        "built-in module"
                      else
                        path
@@ -62,7 +62,7 @@ module Bolt
 
       def print_plan_info(plan)
         path = plan.delete('module')
-        plan['module_dir'] = if path.start_with?(Bolt::PAL::MODULES_PATH)
+        plan['module_dir'] = if path.start_with?(Bolt::Config::Modulepath::MODULES_PATH)
                                "built-in module"
                              else
                                path
@@ -97,13 +97,22 @@ module Bolt
       def print_puppetfile_result(success, puppetfile, moduledir)
         @stream.puts({ "success": success,
                        "puppetfile": puppetfile,
-                       "moduledir": moduledir }.to_json)
+                       "moduledir": moduledir.to_s }.to_json)
       end
 
-      def print_targets(targets)
+      def print_targets(target_list, inventoryfile)
         @stream.puts ::JSON.pretty_generate(
-          "targets": targets.map(&:name),
-          "count": targets.count
+          "inventory": {
+            "targets": target_list[:inventory].map(&:name),
+            "count": target_list[:inventory].count,
+            "file": inventoryfile.to_s
+          },
+          "adhoc": {
+            "targets": target_list[:adhoc].map(&:name),
+            "count": target_list[:adhoc].count
+          },
+          "targets": target_list.values.flatten.map(&:name),
+          "count": target_list.values.flatten.count
         )
       end
 
@@ -135,6 +144,12 @@ module Bolt
       def print_message(message)
         $stderr.puts(message)
       end
+      alias print_error print_message
+
+      def print_action_step(step)
+        $stderr.puts(step)
+      end
+      alias print_action_error print_action_step
     end
   end
 end

data/lib/bolt/outputter/logger.rb

@@ -13,9 +13,9 @@ module Bolt
       def handle_event(event)
         case event[:type]
         when :step_start
-          log_step_start(event)
+          log_step_start(**event)
         when :step_finish
-          log_step_finish(event)
+          log_step_finish(**event)
         when :plan_start
           log_plan_start(event)
         when :plan_finish

data/lib/bolt/pal.rb

@@ -5,18 +5,16 @@ require 'bolt/executor'
 require 'bolt/error'
 require 'bolt/plan_result'
 require 'bolt/util'
+require 'bolt/config/modulepath'
 require 'etc'
 
 module Bolt
   class PAL
-    BOLTLIB_PATH = File.expand_path('../../bolt-modules', __dir__)
-    MODULES_PATH = File.expand_path('../../modules', __dir__)
-
     # PALError is used to convert errors from executing puppet code into
     # Bolt::Errors
     class PALError < Bolt::Error
       def self.from_preformatted_error(err)
-        if err.cause&.is_a? Bolt::Error
+        if err.cause.is_a? Bolt::Error
           err.cause
         else
           from_error(err)
@@ -29,15 +27,12 @@ module Bolt
         message = err.cause ? err.cause.message : err.message
 
         # Provide the location of an error if it came from a plan
-        details = if defined?(err.file) && err.file
-                    { file:   err.file,
-                      line:   err.line,
-                      column: err.pos }.compact
-                  else
-                    {}
-                  end
+        details = {}
+        details[:file]   = err.file if defined?(err.file)
+        details[:line]   = err.line if defined?(err.line)
+        details[:column] = err.pos if defined?(err.pos)
 
-        e = new(message, details)
+        e = new(message, details.compact)
 
         e.set_backtrace(err.backtrace)
         e
@@ -48,16 +43,16 @@ module Bolt
       end
     end
 
-    attr_reader :modulepath, :user_modulepath
-
     def initialize(modulepath, hiera_config, resource_types, max_compiles = Etc.nprocessors,
                    trusted_external = nil, apply_settings = {}, project = nil)
+      unless modulepath.is_a?(Bolt::Config::Modulepath)
+        msg = "Type error in PAL: modulepath must be a Bolt::Config::Modulepath"
+        raise Bolt::Error.new(msg, "bolt/execution-error")
+      end
       # Nothing works without initialized this global state. Reinitializing
       # is safe and in practice only happens in tests
       self.class.load_puppet
-
-      @user_modulepath = modulepath
-      @modulepath = [BOLTLIB_PATH, *modulepath, MODULES_PATH]
+      @modulepath = modulepath
       @hiera_config = hiera_config
       @trusted_external = trusted_external
       @apply_settings = apply_settings
@@ -66,13 +61,21 @@ module Bolt
       @project = project
 
       @logger = Bolt::Logger.logger(self)
-      if modulepath && !modulepath.empty?
-        @logger.debug("Loading modules from #{@modulepath.join(File::PATH_SEPARATOR)}")
+      unless user_modulepath.empty?
+        @logger.debug("Loading modules from #{full_modulepath.join(File::PATH_SEPARATOR)}")
       end
 
       @loaded = false
     end
 
+    def full_modulepath
+      @modulepath.full_modulepath
+    end
+
+    def user_modulepath
+      @modulepath.user_modulepath
+    end
+
     # Puppet logging is global so this is class method to avoid confusion
     def self.configure_logging
       Puppet::Util::Log.destinations.clear
@@ -141,16 +144,33 @@ module Bolt
       end
     end
 
+    def detect_project_conflict(project, environment)
+      return unless project && project.load_as_module?
+      # The environment modulepath has stripped out non-existent directories,
+      # so we don't need to check for them
+      modules = environment.modulepath.flat_map do |path|
+        Dir.children(path).select { |name| Puppet::Module.is_module_directory?(name, path) }
+      end
+      if modules.include?(project.name)
+        Bolt::Logger.warn_once("project shadows module",
+                               "The project '#{project.name}' shadows an existing module of the same name")
+      end
+    end
+
     # Runs a block in a PAL script compiler configured for Bolt.  Catches
     # exceptions thrown by the block and re-raises them ensuring they are
     # Bolt::Errors since the script compiler block will squash all exceptions.
     def in_bolt_compiler
       # TODO: If we always call this inside a bolt_executor we can remove this here
       setup
-      r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
+      r = Puppet::Pal.in_tmp_environment('bolt', modulepath: full_modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
         Puppet.override(bolt_project: @project,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
+          # Because this has the side effect of loading and caching the list
+          # of modules, it must happen *after* we have overridden
+          # bolt_project or the project will be ignored
+          detect_project_conflict(@project, Puppet.lookup(:environments).get('bolt'))
           pal.with_script_compiler(set_local_facts: false) do |compiler|
             alias_types(compiler)
             register_resource_types(Puppet.lookup(:loaders)) if @resource_types
@@ -198,11 +218,11 @@ module Bolt
         apply_executor: applicator || Applicator.new(
           inventory,
           executor,
-          @modulepath,
+          full_modulepath,
           # Skip syncing built-in plugins, since we vendor some Puppet 6
           # versions of "core" types, which are already present on the agent,
           # but may cause issues on Puppet 5 agents.
-          @user_modulepath,
+          user_modulepath,
           @project,
           pdb_client,
           @hiera_config,
@@ -424,13 +444,14 @@ module Bolt
     # Returns a mapping of all modules available to the Bolt compiler
     #
     # @return [Hash{String => Array<Hash{Symbol => String,nil}>}]
-    #   A hash that associates each directory on the module path with an array
+    #   A hash that associates each directory on the modulepath with an array
     #   containing a hash of information for each module in that directory.
     #   The information hash provides the name, version, and a string
     #   indicating whether the module belongs to an internal module group.
     def list_modules
-      internal_module_groups = { BOLTLIB_PATH => 'Plan Language Modules',
-                                 MODULES_PATH => 'Packaged Modules' }
+      internal_module_groups = { Bolt::Config::Modulepath::BOLTLIB_PATH => 'Plan Language Modules',
+                                 Bolt::Config::Modulepath::MODULES_PATH => 'Packaged Modules',
+                                 @project.managed_moduledir.to_s => 'Project Dependencies' }
 
       in_bolt_compiler do
         # NOTE: Can replace map+to_h with transform_values when Ruby 2.4

data/lib/bolt/plugin.rb

@@ -168,7 +168,7 @@ module Bolt
     end
 
     def modules
-      @modules ||= Bolt::Module.discover(@pal.modulepath)
+      @modules ||= Bolt::Module.discover(@pal.full_modulepath)
     end
 
     def add_plugin(plugin)

data/lib/bolt/plugin/module.rb

@@ -16,7 +16,7 @@ module Bolt
         mod = modules[name]
         if mod&.plugin?
           opts[:mod] = mod
-          plugin = Bolt::Plugin::Module.new(opts)
+          plugin = Bolt::Plugin::Module.new(**opts)
           plugin.setup
           plugin
         else

data/lib/bolt/project.rb

@@ -16,9 +16,10 @@ module Bolt
                  "These tasks are included in `bolt task show` output"
     }.freeze
 
-    attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
+    attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
-                :deprecations, :downloads, :plans_path
+                :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
+                :backup_dir
 
     def self.default_project(logs = [])
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
@@ -50,6 +51,20 @@ module Bolt
     def self.create_project(path, type = 'option', logs = [])
       fullpath = Pathname.new(path).expand_path
 
+      if type == 'user'
+        begin
+          # This is already expanded if the type is user
+          FileUtils.mkdir_p(path)
+        rescue StandardError
+          logs << { warn: "Could not create default project at #{path}. Continuing without a writeable project. "\
+                    "Log and rerun files will not be written." }
+        end
+      end
+
+      if type == 'option' && !File.directory?(path)
+        raise Bolt::Error.new("Could not find project at #{path}", "bolt/project-error")
+      end
+
       if !Bolt::Util.windows? && type != 'environment' && fullpath.world_writable?
         raise Bolt::Error.new(
           "Project directory '#{fullpath}' is world-writable which poses a security risk. Set "\
@@ -80,15 +95,16 @@ module Bolt
         @deprecations << { type: 'Using bolt.yaml for project configuration', msg: msg }
       end
 
-      @inventory_file = @path + 'inventory.yaml'
-      @modulepath = [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
-      @hiera_config = @path + 'hiera.yaml'
-      @puppetfile = @path + 'Puppetfile'
-      @rerunfile = @path + '.rerun.json'
-      @resource_types = @path + '.resource_types'
-      @type = type
-      @downloads = @path + 'downloads'
-      @plans_path = @path + 'plans'
+      @inventory_file    = @path + 'inventory.yaml'
+      @hiera_config      = @path + 'hiera.yaml'
+      @puppetfile        = @path + 'Puppetfile'
+      @rerunfile         = @path + '.rerun.json'
+      @resource_types    = @path + '.resource_types'
+      @type              = type
+      @downloads         = @path + 'downloads'
+      @plans_path        = @path + 'plans'
+      @managed_moduledir = @path + '.modules'
+      @backup_dir        = @path + '.bolt-bak'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
@@ -98,6 +114,14 @@ module Bolt
 
       @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_OPTIONS.include?(k) }
 
+      # If the 'modules' key is present in the project configuration file,
+      # use the new, shorter modulepath.
+      @modulepath = if @data.key?('modules')
+                      [(@path + 'modules').to_s]
+                    else
+                      [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
+                    end
+
       # Once bolt.yaml deprecation is removed, this attribute should be removed
       # and replaced with .project_file in lib/bolt/config.rb
       @config_file = if (Bolt::Config::BOLT_OPTIONS & @data.keys).any?
@@ -149,6 +173,16 @@ module Bolt
       @data['plans']
     end
 
+    def modules
+      @modules ||= @data['modules']&.map do |mod|
+        if mod.is_a?(String)
+          { 'name' => mod }
+        else
+          mod
+        end
+      end
+    end
+
     def validate
       if name
         if name !~ Bolt::Module::MODULE_NAME_REGEX
@@ -156,7 +190,7 @@ module Bolt
           Invalid project name '#{name}' in bolt-project.yaml; project name must begin with a lowercase letter
           and can include lowercase letters, numbers, and underscores.
           ERROR_STRING
-        elsif Dir.children(Bolt::PAL::BOLTLIB_PATH).include?(name)
+        elsif Dir.children(Bolt::Config::Modulepath::BOLTLIB_PATH).include?(name)
           raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
             "with a built-in Bolt module of the same name."
         end
@@ -170,6 +204,22 @@ module Bolt
           raise Bolt::ValidationError, "'#{conf}' in bolt-project.yaml must be an array"
         end
       end
+
+      if @data['modules']
+        unless @data['modules'].is_a?(Array)
+          raise Bolt::ValidationError, "'modules' in bolt-project.yaml must be an array"
+        end
+
+        @data['modules'].each do |mod|
+          next if (mod.is_a?(Hash) && mod.key?('name')) || mod.is_a?(String)
+          raise Bolt::ValidationError, "Module declaration #{mod.inspect} must be a hash with a name key"
+        end
+
+        unknown_keys = modules.flat_map(&:keys).uniq - %w[name version_requirement]
+        if unknown_keys.any?
+          @logs << { warn: "Ignoring unknown keys in module declarations: #{unknown_keys.join(', ')}." }
+        end
+      end
     end
 
     def check_deprecated_file