bolt 2.14.0 → 2.19.0

data/bolt-modules/file/lib/puppet/functions/file/write.rb

@@ -15,7 +15,9 @@ Puppet::Functions.create_function(:'file::write') do
   end
 
   def write(filename, content)
+    # Send Analytics Report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     File.write(filename, content)
     nil
   end

data/bolt-modules/out/lib/puppet/functions/out/message.rb

@@ -23,7 +23,9 @@ Puppet::Functions.create_function(:'out::message') do
     end
 
     executor = Puppet.lookup(:bolt_executor)
+    # Send Analytics Report
     executor.report_function_call(self.class.name)
+
     executor.publish_event(type: :message, message: message)
 
     nil

data/bolt-modules/prompt/lib/puppet/functions/prompt.rb

@@ -30,6 +30,7 @@ Puppet::Functions.create_function(:prompt) do
     options = options.transform_keys(&:to_sym)
 
     executor = Puppet.lookup(:bolt_executor)
+    # Send analytics report
     executor.report_function_call(self.class.name)
 
     response = executor.prompt(prompt, options)

data/bolt-modules/system/lib/puppet/functions/system/env.rb

@@ -12,7 +12,9 @@ Puppet::Functions.create_function(:'system::env') do
   end
 
   def env(name)
+    # Send analytics report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     ENV[name]
   end
 end

data/lib/bolt/applicator.rb

@@ -18,7 +18,6 @@ module Bolt
                    pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
-
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath || []
@@ -30,17 +29,6 @@ module Bolt
 
       @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles)
       @logger = Logging.logger[self]
-      @plugin_tarball = Concurrent::Delay.new do
-        build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs << mod.files if mod.files?
-          type_files = "#{mod.path}/types"
-          search_dirs << type_files if File.exist?(type_files)
-          search_dirs
-        end
-      end
     end
 
     private def libexec
@@ -106,6 +94,16 @@ module Bolt
       out, err, stat = Open3.capture3('ruby', bolt_catalog_exe, 'compile', stdin_data: catalog_input.to_json)
       ENV['PATH'] = old_path
 
+      # If bolt_catalog does not return valid JSON, we should print stderr to
+      # see what happened
+      print_logs = stat.success?
+      result = begin
+                 JSON.parse(out)
+               rescue JSON::ParserError
+                 print_logs = true
+                 { 'message' => "Something's gone terribly wrong! STDERR is logged." }
+               end
+
       # Any messages logged by Puppet will be on stderr as JSON hashes, so we
       # parse those and store them here. Any message on stderr that is not
       # properly JSON formatted is assumed to be an error message.  If
@@ -119,17 +117,15 @@ module Bolt
         { 'level' => 'err', 'message' => line }
       end
 
-      result = JSON.parse(out)
-      if stat.success?
+      if print_logs
         logs.each do |log|
           bolt_level = Bolt::Util::PuppetLogLevel::MAPPING[log['level'].to_sym]
           message = log['message'].chomp
           @logger.send(bolt_level, "#{target.name}: #{message}")
         end
-        result
-      else
-        raise ApplyError.new(target.name, result['message'])
       end
+      raise ApplyError.new(target.name, result['message']) unless stat.success?
+      result
     end
 
     def validate_hiera_config(hiera_config)
@@ -180,7 +176,6 @@ module Bolt
 
     def apply_ast(raw_ast, targets, options, plan_vars = {})
       ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
-
       # Serialize as pcore for *Result* objects
       plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
                                                                        rich_data: true,
@@ -188,19 +183,37 @@ module Bolt
                                                                        type_by_reference: true,
                                                                        local_reference: true)
 
+      bolt_project = @project if @project&.name
       scope = {
         code_ast: ast,
         modulepath: @modulepath,
-        project: @project.to_h,
+        project: bolt_project.to_h,
         pdb_config: @pdb_client.config.to_hash,
         hiera_config: @hiera_config,
         plan_vars: plan_vars,
         # This data isn't available on the target config hash
         config: @inventory.transport_data_get
       }
-
       description = options[:description] || 'apply catalog'
 
+      required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+      if required_modules&.any?
+        @logger.debug("Syncing only required modules: #{required_modules.join(',')}.")
+      end
+
+      @plugin_tarball = Concurrent::Delay.new do
+        build_plugin_tarball do |mod|
+          next unless required_modules.nil? || required_modules.include?(mod.name)
+          search_dirs = []
+          search_dirs << mod.plugins if mod.plugins?
+          search_dirs << mod.pluginfacts if mod.pluginfacts?
+          search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
+          search_dirs
+        end
+      end
+
       r = @executor.log_action(description, targets) do
         futures = targets.map do |target|
           Concurrent::Future.execute(executor: @pool) do
@@ -227,6 +240,7 @@ module Bolt
                   result
                 end
               else
+
                 arguments = {
                   'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
                   'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),

data/lib/bolt/apply_result.rb

@@ -20,7 +20,7 @@ module Bolt
              error_hash['msg'] =~ /The term 'ruby.exe' is not recognized as the name of a cmdlet/)
         # Windows does not have Ruby present
         {
-          'msg' => "Puppet is not installed on the target in $env:ProgramFiles, please install it to enable 'apply'",
+          'msg' => "Puppet was not found on the target or in $env:ProgramFiles, please install it to enable 'apply'",
           'kind' => 'bolt/apply-error'
         }
       elsif exit_code == 1 && error_hash['msg'] =~ /cannot load such file -- puppet \(LoadError\)/

data/lib/bolt/bolt_option_parser.rb

@@ -10,10 +10,10 @@ module Bolt
                 authentication: %w[user password password-prompt private-key host-key-check ssl ssl-verify],
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
-                global_config_setters: %w[modulepath boltdir configfile],
-                transports: %w[transport connect-timeout tty ssh-command copy-command],
+                global_config_setters: %w[modulepath project configfile],
+                transports: %w[transport connect-timeout tty native-ssh ssh-command copy-command],
                 display: %w[format color verbose trace],
-                global: %w[help version debug] }.freeze
+                global: %w[help version debug log-level] }.freeze
 
     ACTION_OPTS = OPTIONS.values.flatten.freeze
 
@@ -422,7 +422,7 @@ module Bolt
 
       ACTIONS
           generate-types        Generate type references to register in plans
-          install               Install modules from a Puppetfile into a Boltdir
+          install               Install modules from a Puppetfile into a project
           show-modules          List modules available to the Bolt project
     HELP
 
@@ -445,7 +445,7 @@ module Bolt
           bolt puppetfile install [options]
 
       DESCRIPTION
-          Install modules from a Puppetfile into a Boltdir
+          Install modules from a Puppetfile into a project
     HELP
 
     PUPPETFILE_SHOWMODULES_HELP = <<~HELP
@@ -594,13 +594,13 @@ module Bolt
             bolt task show canary
     HELP
 
-    attr_reader :warnings
+    attr_reader :deprecations
 
     def initialize(options)
       super()
 
       @options = options
-      @warnings = []
+      @deprecations = []
 
       separator "\nINVENTORY OPTIONS"
       define('-t', '--targets TARGETS',
@@ -709,29 +709,29 @@ module Bolt
           File.expand_path(moduledir)
         end
       end
-      define('--boltdir FILEPATH',
-             'Specify what Boltdir to load config from (default: autodiscovered from current working dir)') do |path|
+      define('--project PATH', '--boltdir PATH',
+             'Specify what project to load config from (default: autodiscovered from current working dir)') do |path|
         @options[:boltdir] = path
       end
-      define('--configfile FILEPATH',
+      define('--configfile PATH',
              'Specify where to load config from (default: ~/.puppetlabs/bolt/bolt.yaml).',
-             'Directory containing bolt.yaml will be used as the Boltdir.') do |path|
+             'Directory containing bolt.yaml will be used as the project directory.') do |path|
         @options[:configfile] = path
       end
-      define('--hiera-config FILEPATH',
+      define('--hiera-config PATH',
              'Specify where to load Hiera config from (default: ~/.puppetlabs/bolt/hiera.yaml)') do |path|
         @options[:'hiera-config'] = File.expand_path(path)
       end
-      define('-i', '--inventoryfile FILEPATH',
+      define('-i', '--inventoryfile PATH',
              'Specify where to load inventory from (default: ~/.puppetlabs/bolt/inventory.yaml)') do |path|
         if ENV.include?(Bolt::Inventory::ENVIRONMENT_VAR)
           raise Bolt::CLIError, "Cannot pass inventory file when #{Bolt::Inventory::ENVIRONMENT_VAR} is set"
         end
         @options[:inventoryfile] = Pathname.new(File.expand_path(path))
       end
-      define('--puppetfile FILEPATH',
+      define('--puppetfile PATH',
              'Specify a Puppetfile to use when installing modules. (default: ~/.puppetlabs/bolt/Puppetfile)',
-             'Modules are installed in the current Boltdir.') do |path|
+             'Modules are installed in the current project.') do |path|
         @options[:puppetfile_path] = Pathname.new(File.expand_path(path))
       end
       define('--[no-]save-rerun', 'Whether to update the rerun file after this command.') do |save|
@@ -743,11 +743,15 @@ module Bolt
              "Specify a default transport: #{TRANSPORTS.keys.join(', ')}") do |t|
         @options[:transport] = t
       end
-      define('--ssh-command EXEC', "Executable to use instead of the net-ssh ruby library. ",
+      define('--[no-]native-ssh', 'Whether to shell out to native SSH or use the net-ssh Ruby library.',
+             'This option is experimental') do |bool|
+        @options[:'native-ssh'] = bool
+      end
+      define('--ssh-command EXEC', "Executable to use instead of the net-ssh Ruby library. ",
              "This option is experimental.") do |exec|
         @options[:'ssh-command'] = exec
       end
-      define('--copy-command EXEC', "Command to copy files to remote hosts if using external SSH. ",
+      define('--copy-command EXEC', "Command to copy files to remote hosts if using native SSH. ",
              "This option is experimental.") do |exec|
         @options[:'copy-command'] = exec
       end
@@ -803,8 +807,16 @@ module Bolt
       end
       define('--debug', 'Display debug logging') do |_|
         @options[:debug] = true
+        # We don't actually set '--log-level debug' here, but once the options are evaluated by
+        # the config class the end result is the same.
+        msg = "Command line option '--debug' is deprecated, set '--log-level debug' instead."
+        @deprecations << { type: 'Using --debug instead of --log-level debug', msg: msg }
+      end
+      define('--log-level LEVEL',
+             "Set the log level for the console. Available options are",
+             "debug, info, notice, warn, error, fatal, any.") do |level|
+        @options[:log] = { 'console' => { 'level' => level } }
       end
-
       define('--plugin PLUGIN', 'Select the plugin to use') do |plug|
         @options[:plugin] = plug
       end

data/lib/bolt/cli.rb

@@ -46,7 +46,6 @@ module Bolt
       Bolt::Logger.initialize_logging
       @logger = Logging.logger[self]
       @argv = argv
-      @config = Bolt::Config.default
       @options = {}
     end
 
@@ -77,63 +76,83 @@ module Bolt
     private :help?
 
     def parse
-      parser = BoltOptionParser.new(options)
-      # This part aims to handle both `bolt <mode> --help` and `bolt help <mode>`.
-      remaining = handle_parser_errors { parser.permute(@argv) } unless @argv.empty?
-      if @argv.empty? || help?(remaining)
-        # Update the parser for the subcommand (or lack thereof)
-        parser.update
-        puts parser.help
-        raise Bolt::CLIExit
-      end
-
-      options[:object] = remaining.shift
-
-      # Only parse task_options for task or plan
-      if %w[task plan].include?(options[:subcommand])
-        task_options, remaining = remaining.partition { |s| s =~ /.+=/ }
-        if options[:task_options]
-          unless task_options.empty?
-            raise Bolt::CLIError,
-                  "Parameters must be specified through either the --params " \
-                  "option or param=value pairs, not both"
+      begin
+        parser = BoltOptionParser.new(options)
+        # This part aims to handle both `bolt <mode> --help` and `bolt help <mode>`.
+        remaining = handle_parser_errors { parser.permute(@argv) } unless @argv.empty?
+        if @argv.empty? || help?(remaining)
+          # Update the parser for the subcommand (or lack thereof)
+          parser.update
+          puts parser.help
+          raise Bolt::CLIExit
+        end
+
+        options[:object] = remaining.shift
+
+        # Only parse task_options for task or plan
+        if %w[task plan].include?(options[:subcommand])
+          task_options, remaining = remaining.partition { |s| s =~ /.+=/ }
+          if options[:task_options]
+            unless task_options.empty?
+              raise Bolt::CLIError,
+                    "Parameters must be specified through either the --params " \
+                    "option or param=value pairs, not both"
+            end
+            options[:params_parsed] = true
+          elsif task_options.any?
+            options[:params_parsed] = false
+            options[:task_options] = Hash[task_options.map { |a| a.split('=', 2) }]
+          else
+            options[:params_parsed] = true
+            options[:task_options] = {}
           end
-          options[:params_parsed] = true
-        elsif task_options.any?
-          options[:params_parsed] = false
-          options[:task_options] = Hash[task_options.map { |a| a.split('=', 2) }]
+        end
+        options[:leftovers] = remaining
+
+        validate(options)
+
+        @config = if ENV['BOLT_PROJECT']
+                    project = Bolt::Project.create_project(ENV['BOLT_PROJECT'], 'environment')
+                    Bolt::Config.from_project(project, options)
+                  elsif options[:configfile]
+                    Bolt::Config.from_file(options[:configfile], options)
+                  else
+                    project = if options[:boltdir]
+                                dir = Pathname.new(options[:boltdir])
+                                if (dir + Bolt::Project::BOLTDIR_NAME).directory?
+                                  Bolt::Project.create_project(dir + Bolt::Project::BOLTDIR_NAME)
+                                else
+                                  Bolt::Project.create_project(dir)
+                                end
+                              else
+                                Bolt::Project.find_boltdir(Dir.pwd)
+                              end
+                    Bolt::Config.from_project(project, options)
+                  end
+
+        Bolt::Logger.configure(config.log, config.color)
+        Bolt::Logger.analytics = analytics
+      rescue Bolt::Error => e
+        if $stdout.isatty
+          # Print the error message in red, mimicking outputter.fatal_error
+          $stdout.puts("\033[31m#{e.message}\033[0m")
         else
-          options[:params_parsed] = true
-          options[:task_options] = {}
+          $stdout.puts(e.message)
         end
+        raise e
       end
-      options[:leftovers] = remaining
-
-      validate(options)
-
-      @config = if options[:configfile]
-                  Bolt::Config.from_file(options[:configfile], options)
-                else
-                  project = if options[:boltdir]
-                              Bolt::Project.create_project(options[:boltdir])
-                            else
-                              Bolt::Project.find_boltdir(Dir.pwd)
-                            end
-                  Bolt::Config.from_project(project, options)
-                end
-
-      Bolt::Logger.configure(config.log, config.color)
 
       # Logger must be configured before checking path case and project file, otherwise warnings will not display
-      @config.check_path_case('modulepath', @config.modulepath)
-      @config.project.check_deprecated_file
+      config.check_path_case('modulepath', config.modulepath)
+      config.project.check_deprecated_file
 
       # Log the file paths for loaded config files
       config_loaded
 
       # Display warnings created during parser and config initialization
-      parser.warnings.each { |warning| @logger.warn(warning[:msg]) }
       config.warnings.each { |warning| @logger.warn(warning[:msg]) }
+      parser.deprecations.each { |dep| Bolt::Logger.deprecation_warning(dep[:type], dep[:msg]) }
+      config.deprecations.each { |dep| Bolt::Logger.deprecation_warning(dep[:type], dep[:msg]) }
 
       # After validation, initialize inventory and targets. Errors here are better to catch early.
       # After this step
@@ -219,7 +238,7 @@ module Bolt
       end
 
       if options[:boltdir] && options[:configfile]
-        raise Bolt::CLIError, "Only one of '--boltdir' or '--configfile' may be specified"
+        raise Bolt::CLIError, "Only one of '--boltdir', '--project', or '--configfile' may be specified"
       end
 
       if options[:noop] &&
@@ -245,6 +264,10 @@ module Bolt
          !options[:object]
         raise Bolt::CLIError, "Must specify a value to #{options[:action]}"
       end
+
+      if options.key?(:debug) && options.key?(:log)
+        raise Bolt::CLIError, "Only one of '--debug' or '--log-level' may be specified"
+      end
     end
 
     def handle_parser_errors
@@ -272,12 +295,12 @@ module Bolt
     def warn_inventory_overrides_cli(opts)
       inventory_source = if ENV[Bolt::Inventory::ENVIRONMENT_VAR]
                            Bolt::Inventory::ENVIRONMENT_VAR
-                         elsif @config.inventoryfile && Bolt::Util.file_stat(@config.inventoryfile)
-                           @config.inventoryfile
+                         elsif config.inventoryfile && Bolt::Util.file_stat(config.inventoryfile)
+                           config.inventoryfile
                          else
                            begin
-                             Bolt::Util.file_stat(@config.default_inventoryfile)
-                             @config.default_inventoryfile
+                             Bolt::Util.file_stat(config.default_inventoryfile)
+                             config.default_inventoryfile
                            rescue Errno::ENOENT
                              nil
                            end
@@ -381,7 +404,7 @@ module Bolt
         if options[:action] == 'generate-types'
           code = generate_types
         elsif options[:action] == 'install'
-          code = install_puppetfile(@config.puppetfile_config, @config.puppetfile, @config.modulepath)
+          code = install_puppetfile(config.puppetfile_config, config.puppetfile, config.modulepath)
         end
       when 'secret'
         code = Bolt::Secret.execute(plugins, outputter, options)
@@ -513,7 +536,7 @@ module Bolt
       plan_context[:description] = options[:description] if options[:description]
 
       executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop], config.modified_concurrency)
-      if options.fetch(:format, 'human') == 'human'
+      if %w[human rainbow].include?(options.fetch(:format, 'human'))
         executor.subscribe(outputter)
       else
         # Only subscribe to out::message events for JSON outputter
@@ -771,14 +794,13 @@ module Bolt
     end
 
     def pal
-      project = config.project.project_file? ? config.project : nil
       @pal ||= Bolt::PAL.new(config.modulepath,
                              config.hiera_config,
                              config.project.resource_types,
                              config.compile_concurrency,
                              config.trusted_external,
                              config.apply_settings,
-                             project)
+                             config.project)
     end
 
     def convert_plan(plan)
@@ -794,7 +816,7 @@ module Bolt
     end
 
     def rerun
-      @rerun ||= Bolt::Rerun.new(@config.rerunfile, @config.save_rerun)
+      @rerun ||= Bolt::Rerun.new(config.rerunfile, config.save_rerun)
     end
 
     def outputter