bolt 2.13.0 → 2.18.0

data/bolt-modules/file/lib/puppet/functions/file/join.rb

@@ -12,7 +12,9 @@ Puppet::Functions.create_function(:'file::join') do
   end
 
   def join(*paths)
+    # Send Analytics Report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     File.join(paths)
   end
 end

data/bolt-modules/file/lib/puppet/functions/file/read.rb

@@ -8,7 +8,7 @@ Puppet::Functions.create_function(:'file::read', Puppet::Functions::InternalFunc
   # @example Read a file from disk
   #   file::read('/tmp/i_dumped_this_here')
   # @example Read a file from the modulepath
-  #   file::read('example/files/VERSION')
+  #   file::read('example/VERSION')
   dispatch :read do
     scope_param
     required_param 'String', :filename
@@ -16,7 +16,9 @@ Puppet::Functions.create_function(:'file::read', Puppet::Functions::InternalFunc
   end
 
   def read(scope, filename)
+    # Send Analytics Report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     found = Puppet::Parser::Files.find_file(filename, scope.compiler.environment)
     unless found && Puppet::FileSystem.exist?(found)
       raise Puppet::ParseErrorWithIssue.from_issue_and_stack(

data/bolt-modules/file/lib/puppet/functions/file/readable.rb

@@ -9,7 +9,7 @@ Puppet::Functions.create_function(:'file::readable', Puppet::Functions::Internal
   # @example Check a file on disk
   #   file::readable('/tmp/i_dumped_this_here')
   # @example check a file from the modulepath
-  #   file::readable('example/files/VERSION')
+  #   file::readable('example/VERSION')
   dispatch :readable do
     scope_param
     required_param 'String', :filename
@@ -17,7 +17,9 @@ Puppet::Functions.create_function(:'file::readable', Puppet::Functions::Internal
   end
 
   def readable(scope, filename)
+    # Send Analytics Report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     found = Puppet::Parser::Files.find_file(filename, scope.compiler.environment)
     found ? File.readable?(found) : false
   end

data/bolt-modules/file/lib/puppet/functions/file/write.rb

@@ -15,7 +15,9 @@ Puppet::Functions.create_function(:'file::write') do
   end
 
   def write(filename, content)
+    # Send Analytics Report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     File.write(filename, content)
     nil
   end

data/bolt-modules/out/lib/puppet/functions/out/message.rb

@@ -23,7 +23,9 @@ Puppet::Functions.create_function(:'out::message') do
     end
 
     executor = Puppet.lookup(:bolt_executor)
+    # Send Analytics Report
     executor.report_function_call(self.class.name)
+
     executor.publish_event(type: :message, message: message)
 
     nil

data/bolt-modules/prompt/lib/puppet/functions/prompt.rb

@@ -30,6 +30,7 @@ Puppet::Functions.create_function(:prompt) do
     options = options.transform_keys(&:to_sym)
 
     executor = Puppet.lookup(:bolt_executor)
+    # Send analytics report
     executor.report_function_call(self.class.name)
 
     response = executor.prompt(prompt, options)

data/bolt-modules/system/lib/puppet/functions/system/env.rb

@@ -12,7 +12,9 @@ Puppet::Functions.create_function(:'system::env') do
   end
 
   def env(name)
+    # Send analytics report
     Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+
     ENV[name]
   end
 end

data/lib/bolt/applicator.rb

@@ -18,10 +18,9 @@ module Bolt
                    pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
-
       @inventory = inventory
       @executor = executor
-      @modulepath = modulepath
+      @modulepath = modulepath || []
       @plugin_dirs = plugin_dirs
       @project = project
       @pdb_client = pdb_client
@@ -30,17 +29,6 @@ module Bolt
 
       @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles)
       @logger = Logging.logger[self]
-      @plugin_tarball = Concurrent::Delay.new do
-        build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs << mod.files if mod.files?
-          type_files = "#{mod.path}/types"
-          search_dirs << type_files if File.exist?(type_files)
-          search_dirs
-        end
-      end
     end
 
     private def libexec
@@ -106,6 +94,16 @@ module Bolt
       out, err, stat = Open3.capture3('ruby', bolt_catalog_exe, 'compile', stdin_data: catalog_input.to_json)
       ENV['PATH'] = old_path
 
+      # If bolt_catalog does not return valid JSON, we should print stderr to
+      # see what happened
+      print_logs = stat.success?
+      result = begin
+                 JSON.parse(out)
+               rescue JSON::ParserError
+                 print_logs = true
+                 { 'message' => "Something's gone terribly wrong! STDERR is logged." }
+               end
+
       # Any messages logged by Puppet will be on stderr as JSON hashes, so we
       # parse those and store them here. Any message on stderr that is not
       # properly JSON formatted is assumed to be an error message.  If
@@ -119,17 +117,15 @@ module Bolt
         { 'level' => 'err', 'message' => line }
       end
 
-      result = JSON.parse(out)
-      if stat.success?
+      if print_logs
         logs.each do |log|
           bolt_level = Bolt::Util::PuppetLogLevel::MAPPING[log['level'].to_sym]
           message = log['message'].chomp
           @logger.send(bolt_level, "#{target.name}: #{message}")
         end
-        result
-      else
-        raise ApplyError.new(target.name, result['message'])
       end
+      raise ApplyError.new(target.name, result['message']) unless stat.success?
+      result
     end
 
     def validate_hiera_config(hiera_config)
@@ -146,6 +142,7 @@ module Bolt
 
     def apply(args, apply_body, scope)
       raise(ArgumentError, 'apply requires a TargetSpec') if args.empty?
+      raise(ArgumentError, 'apply requires at least one statement in the apply block') if apply_body.nil?
       type0 = Puppet.lookup(:pal_script_compiler).type('TargetSpec')
       Puppet::Pal.assert_type(type0, args[0], 'apply targets')
 
@@ -179,7 +176,6 @@ module Bolt
 
     def apply_ast(raw_ast, targets, options, plan_vars = {})
       ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
-
       # Serialize as pcore for *Result* objects
       plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
                                                                        rich_data: true,
@@ -187,19 +183,37 @@ module Bolt
                                                                        type_by_reference: true,
                                                                        local_reference: true)
 
+      bolt_project = @project if @project&.name
       scope = {
         code_ast: ast,
         modulepath: @modulepath,
-        project: @project.to_h,
+        project: bolt_project.to_h,
         pdb_config: @pdb_client.config.to_hash,
         hiera_config: @hiera_config,
         plan_vars: plan_vars,
         # This data isn't available on the target config hash
         config: @inventory.transport_data_get
       }
-
       description = options[:description] || 'apply catalog'
 
+      required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+      if required_modules&.any?
+        @logger.debug("Syncing only required modules: #{required_modules.join(',')}.")
+      end
+
+      @plugin_tarball = Concurrent::Delay.new do
+        build_plugin_tarball do |mod|
+          next unless required_modules.nil? || required_modules.include?(mod.name)
+          search_dirs = []
+          search_dirs << mod.plugins if mod.plugins?
+          search_dirs << mod.pluginfacts if mod.pluginfacts?
+          search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
+          search_dirs
+        end
+      end
+
       r = @executor.log_action(description, targets) do
         futures = targets.map do |target|
           Concurrent::Future.execute(executor: @pool) do
@@ -226,6 +240,7 @@ module Bolt
                   result
                 end
               else
+
                 arguments = {
                   'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
                   'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),

data/lib/bolt/apply_inventory.rb

@@ -70,6 +70,10 @@ module Bolt
       @targets[target.name].features
     end
 
+    def resource(target, type, title)
+      @targets[target.name].resource(type, title)
+    end
+
     def add_to_group(*_params)
       raise InvalidFunctionCall, 'add_to_group'
     end

data/lib/bolt/apply_result.rb

@@ -20,7 +20,7 @@ module Bolt
              error_hash['msg'] =~ /The term 'ruby.exe' is not recognized as the name of a cmdlet/)
         # Windows does not have Ruby present
         {
-          'msg' => "Puppet is not installed on the target in $env:ProgramFiles, please install it to enable 'apply'",
+          'msg' => "Puppet was not found on the target or in $env:ProgramFiles, please install it to enable 'apply'",
           'kind' => 'bolt/apply-error'
         }
       elsif exit_code == 1 && error_hash['msg'] =~ /cannot load such file -- puppet \(LoadError\)/

data/lib/bolt/apply_target.rb

@@ -62,6 +62,10 @@ module Bolt
       @safe_name
     end
 
+    def resource(type, title)
+      resources[Bolt::ResourceInstance.format_reference(type, title)]
+    end
+
     def parse_uri(string)
       require 'addressable/uri'
       if string.nil?

data/lib/bolt/bolt_option_parser.rb

@@ -10,10 +10,10 @@ module Bolt
                 authentication: %w[user password password-prompt private-key host-key-check ssl ssl-verify],
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
-                global_config_setters: %w[modulepath boltdir configfile],
-                transports: %w[transport connect-timeout tty ssh-command copy-command],
+                global_config_setters: %w[modulepath project configfile],
+                transports: %w[transport connect-timeout tty native-ssh ssh-command copy-command],
                 display: %w[format color verbose trace],
-                global: %w[help version debug] }.freeze
+                global: %w[help version debug log-level] }.freeze
 
     ACTION_OPTS = OPTIONS.values.flatten.freeze
 
@@ -422,7 +422,7 @@ module Bolt
 
       ACTIONS
           generate-types        Generate type references to register in plans
-          install               Install modules from a Puppetfile into a Boltdir
+          install               Install modules from a Puppetfile into a project
           show-modules          List modules available to the Bolt project
     HELP
 
@@ -445,7 +445,7 @@ module Bolt
           bolt puppetfile install [options]
 
       DESCRIPTION
-          Install modules from a Puppetfile into a Boltdir
+          Install modules from a Puppetfile into a project
     HELP
 
     PUPPETFILE_SHOWMODULES_HELP = <<~HELP
@@ -709,29 +709,29 @@ module Bolt
           File.expand_path(moduledir)
         end
       end
-      define('--boltdir FILEPATH',
-             'Specify what Boltdir to load config from (default: autodiscovered from current working dir)') do |path|
+      define('--project PATH', '--boltdir PATH',
+             'Specify what project to load config from (default: autodiscovered from current working dir)') do |path|
         @options[:boltdir] = path
       end
-      define('--configfile FILEPATH',
+      define('--configfile PATH',
              'Specify where to load config from (default: ~/.puppetlabs/bolt/bolt.yaml).',
-             'Directory containing bolt.yaml will be used as the Boltdir.') do |path|
+             'Directory containing bolt.yaml will be used as the project directory.') do |path|
         @options[:configfile] = path
       end
-      define('--hiera-config FILEPATH',
+      define('--hiera-config PATH',
              'Specify where to load Hiera config from (default: ~/.puppetlabs/bolt/hiera.yaml)') do |path|
         @options[:'hiera-config'] = File.expand_path(path)
       end
-      define('-i', '--inventoryfile FILEPATH',
+      define('-i', '--inventoryfile PATH',
              'Specify where to load inventory from (default: ~/.puppetlabs/bolt/inventory.yaml)') do |path|
         if ENV.include?(Bolt::Inventory::ENVIRONMENT_VAR)
           raise Bolt::CLIError, "Cannot pass inventory file when #{Bolt::Inventory::ENVIRONMENT_VAR} is set"
         end
         @options[:inventoryfile] = Pathname.new(File.expand_path(path))
       end
-      define('--puppetfile FILEPATH',
+      define('--puppetfile PATH',
              'Specify a Puppetfile to use when installing modules. (default: ~/.puppetlabs/bolt/Puppetfile)',
-             'Modules are installed in the current Boltdir.') do |path|
+             'Modules are installed in the current project.') do |path|
         @options[:puppetfile_path] = Pathname.new(File.expand_path(path))
       end
       define('--[no-]save-rerun', 'Whether to update the rerun file after this command.') do |save|
@@ -743,11 +743,15 @@ module Bolt
              "Specify a default transport: #{TRANSPORTS.keys.join(', ')}") do |t|
         @options[:transport] = t
       end
-      define('--ssh-command EXEC', "Executable to use instead of the net-ssh ruby library. ",
+      define('--[no-]native-ssh', 'Whether to shell out to native SSH or use the net-ssh Ruby library.',
+             'This option is experimental') do |bool|
+        @options[:'native-ssh'] = bool
+      end
+      define('--ssh-command EXEC', "Executable to use instead of the net-ssh Ruby library. ",
              "This option is experimental.") do |exec|
         @options[:'ssh-command'] = exec
       end
-      define('--copy-command EXEC', "Command to copy files to remote hosts if using external SSH. ",
+      define('--copy-command EXEC', "Command to copy files to remote hosts if using native SSH. ",
              "This option is experimental.") do |exec|
         @options[:'copy-command'] = exec
       end
@@ -803,8 +807,15 @@ module Bolt
       end
       define('--debug', 'Display debug logging') do |_|
         @options[:debug] = true
+        # We don't actually set '--log-level debug' here, but once the options are evaluated by
+        # the config class the end result is the same.
+        @warnings << { msg: "Command line option '--debug' is deprecated, set '--log-level debug' instead." }
+      end
+      define('--log-level LEVEL',
+             "Set the log level for the console. Available options are",
+             "debug, info, notice, warn, error, fatal, any.") do |level|
+        @options[:log] = { 'console' => { 'level' => level } }
       end
-
       define('--plugin PLUGIN', 'Select the plugin to use') do |plug|
         @options[:plugin] = plug
       end

data/lib/bolt/catalog.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
+require 'bolt/apply_inventory'
 require 'bolt/apply_target'
 require 'bolt/config'
 require 'bolt/error'
 require 'bolt/inventory'
-require 'bolt/apply_inventory'
 require 'bolt/pal'
 require 'bolt/puppetdb'
 require 'bolt/util'
@@ -19,7 +19,7 @@ module Bolt
       @log_level = log_level
     end
 
-    def with_puppet_settings(hiera_config = {})
+    def with_puppet_settings(overrides = {})
       Dir.mktmpdir('bolt') do |dir|
         cli = []
         Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
@@ -31,7 +31,9 @@ module Bolt
         Puppet.settings.override_default(:vendormoduledir, '')
 
         Puppet.initialize_settings(cli)
-        Puppet.settings[:hiera_config] = hiera_config
+        overrides.each do |setting, value|
+          Puppet.settings[setting] = value
+        end
 
         # Use a special logdest that serializes all log messages and their level to stderr.
         Puppet::Util::Log.newdestination(:stderr)
@@ -54,83 +56,51 @@ module Bolt
     end
 
     def compile_catalog(request)
-      pal_main = request['code_ast'] || request['code_string']
-      target = request['target']
       pdb_client = Bolt::PuppetDB::Client.new(Bolt::PuppetDB::Config.new(request['pdb_config']))
-      options = request['puppet_config'] || {}
       project = request['project'] || {}
       bolt_project = Struct.new(:name, :path).new(project['name'], project['path']) unless project.empty?
-      with_puppet_settings(request['hiera_config']) do
-        Puppet[:rich_data] = true
-        Puppet[:node_name_value] = target['name']
-        env_conf = { modulepath: request['modulepath'] || [],
-                     facts: target['facts'] || {} }
-        env_conf[:variables] = {}
+      inv = Bolt::ApplyInventory.new(request['config'])
+      puppet_overrides = {
+        bolt_pdb_client: pdb_client,
+        bolt_inventory: inv,
+        bolt_project: bolt_project
+      }
+
+      # Facts will be set by the catalog compiler, so we need to ensure
+      # that any plan or target variables with the same name are not
+      # passed into the apply block to avoid a redefinition error.
+      # Filter out plan and target vars separately and raise a Puppet
+      # warning if there are any collisions for either. Puppet warning
+      # is the only way to log a message that will make it back to Bolt
+      # to be printed.
+      target = request['target']
+      plan_vars = shadow_vars('plan', request['plan_vars'], target['facts'])
+      target_vars = shadow_vars('target', target['variables'], target['facts'])
+      topscope_vars = target_vars.merge(plan_vars)
+      env_conf = { modulepath: request['modulepath'],
+                   facts: target['facts'],
+                   variables: topscope_vars }
+
+      puppet_settings = {
+        node_name_value: target['name'],
+        hiera_config: request['hiera_config']
+      }
+
+      with_puppet_settings(puppet_settings) do
         Puppet::Pal.in_tmp_environment('bolt_catalog', env_conf) do |pal|
-          inv = Bolt::ApplyInventory.new(request['config'])
-          Puppet.override(bolt_pdb_client: pdb_client,
-                          bolt_inventory: inv,
-                          bolt_project: bolt_project) do
+          Puppet.override(puppet_overrides) do
             Puppet.lookup(:pal_current_node).trusted_data = target['trusted']
             pal.with_catalog_compiler do |compiler|
-              # Deserializing needs to happen inside the catalog compiler so
-              # loaders are initialized for loading
-              plan_vars = Puppet::Pops::Serialization::FromDataConverter.convert(request['plan_vars'])
-
-              # Facts will be set by the catalog compiler, so we need to ensure
-              # that any plan or target variables with the same name are not
-              # passed into the apply block to avoid a redefinition error.
-              # Filter out plan and target vars separately and raise a Puppet
-              # warning if there are any collisions for either. Puppet warning
-              # is the only way to log a message that will make it back to Bolt
-              # to be printed.
-              pv_collisions, pv_filtered = plan_vars.partition do |k, _|
-                target['facts'].keys.include?(k)
-              end.map(&:to_h)
-              unless pv_collisions.empty?
-                print_pv = pv_collisions.keys.map { |k| "$#{k}" }.join(', ')
-                plural = pv_collisions.keys.length == 1 ? '' : 's'
-                Puppet.warning("Plan variable#{plural} #{print_pv} will be overridden by fact#{plural} " \
-                               "of the same name in the apply block")
-              end
-
-              tv_collisions, tv_filtered = target['variables'].partition do |k, _|
-                target['facts'].keys.include?(k)
-              end.map(&:to_h)
-              unless tv_collisions.empty?
-                print_tv = tv_collisions.keys.map { |k| "$#{k}" }.join(', ')
-                plural = tv_collisions.keys.length == 1 ? '' : 's'
-                Puppet.warning("Target variable#{plural} #{print_tv} " \
-                               "will be overridden by fact#{plural} of the same name in the apply block")
-              end
-
-              pal.send(:add_variables, compiler.send(:topscope), tv_filtered.merge(pv_filtered))
-
+              options = request['puppet_config'] || {}
               # Configure language strictness in the CatalogCompiler. We want Bolt to be able
               # to compile most Puppet 4+ manifests, so we default to allowing deprecated functions.
               Puppet[:strict] = options['strict'] || :warning
               Puppet[:strict_variables] = options['strict_variables'] || false
-              ast = Puppet::Pops::Serialization::FromDataConverter.convert(pal_main)
-              # This will be a Program when running via `bolt apply`, but will
-              # only be a subset of the AST when compiling an apply block in a
-              # plan. In that case, we need to discover the definitions (which
-              # would ordinarily be stored on the Program) and construct a Program object.
-              unless ast.is_a?(Puppet::Pops::Model::Program)
-                # Node definitions must be at the top level of the apply block.
-                # That means the apply body either a) consists of just a
-                # NodeDefinition, b) consists of a BlockExpression which may
-                # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
-                definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
-                                ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
-                              elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
-                                [ast]
-                              else
-                                []
-                              end
-                ast = Puppet::Pops::Model::Factory.PROGRAM(ast, definitions, ast.locator).model
-              end
+
+              pal_main = request['code_ast'] || request['code_string']
+              ast = build_program(pal_main)
               compiler.evaluate(ast)
-              compiler.instance_variable_get(:@internal_compiler).send(:evaluate_ast_node)
+              compiler.evaluate_ast_node
               compiler.compile_additions
               compiler.with_json_encoding(&:encode)
             end
@@ -138,5 +108,45 @@ module Bolt
         end
       end
     end
+
+    # Warn and remove variables that will be shadowed by facts of the same
+    # name, which are set in scope earlier.
+    def shadow_vars(type, vars, facts)
+      collisions, valid = vars.partition do |k, _|
+        facts.include?(k)
+      end
+      if collisions.any?
+        names = collisions.map { |k, _| "$#{k}" }.join(', ')
+        plural = collisions.length == 1 ? '' : 's'
+        Puppet.warning("#{type.capitalize} variable#{plural} #{names} will be overridden by fact#{plural} " \
+                       "of the same name in the apply block")
+      end
+      valid.to_h
+    end
+
+    def build_program(code)
+      ast = Puppet::Pops::Serialization::FromDataConverter.convert(code)
+
+      # This will be a Program when running via `bolt apply`, but will
+      # only be a subset of the AST when compiling an apply block in a
+      # plan. In that case, we need to discover the definitions (which
+      # would ordinarily be stored on the Program) and construct a Program object.
+      if ast.is_a?(Puppet::Pops::Model::Program)
+        ast
+      else
+        # Node definitions must be at the top level of the apply block.
+        # That means the apply body either a) consists of just a
+        # NodeDefinition, b) consists of a BlockExpression which may
+        # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
+        definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
+                        ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
+                      elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
+                        [ast]
+                      else
+                        []
+                      end
+        Puppet::Pops::Model::Factory.PROGRAM(ast, definitions, ast.locator).model
+      end
+    end
   end
 end