bolt 2.7.0 → 2.11.1

data/bolt-modules/file/lib/puppet/functions/file/exists.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# Check if a file exists.
+# Check if a local file exists using Puppet's
+# `Puppet::Parser::Files.find_file()` function. This will only check files that
+# are on the machine Bolt is run on.
 Puppet::Functions.create_function(:'file::exists', Puppet::Functions::InternalFunction) do
   # @param filename Absolute path or Puppet file path.
   # @return Whether the file exists.

data/bolt-modules/file/lib/puppet/functions/file/join.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Join file paths.
+# Join file paths using ruby's `File.join()` function.
 Puppet::Functions.create_function(:'file::join') do
   # @param paths The paths to join.
   # @return The joined file path.

data/bolt-modules/file/lib/puppet/functions/file/read.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-# Read a file and return its contents.
+# Read a file on localhost and return its contents using ruby's `File.read`. This will
+# only read files on the machine you run Bolt on.
 Puppet::Functions.create_function(:'file::read', Puppet::Functions::InternalFunction) do
   # @param filename Absolute path or Puppet file path.
   # @return The file's contents.

data/bolt-modules/file/lib/puppet/functions/file/readable.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# Check if a file is readable.
+# Check if a local file is readable using Puppet's
+# `Puppet::Parser::Files.find_file()` function. This will only check files on the
+# machine you run Bolt on.
 Puppet::Functions.create_function(:'file::readable', Puppet::Functions::InternalFunction) do
   # @param filename Absolute path or Puppet file path.
   # @return Whether the file is readable.

data/bolt-modules/file/lib/puppet/functions/file/write.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
-# Write a string to a file.
+# Write a string to a file on localhost using ruby's `File.write`. This will
+# only write files to the machine you run Bolt on. Use `write_file()` to write
+# to remote targets.
 Puppet::Functions.create_function(:'file::write') do
   # @param filename Absolute path.
   # @param content File content to write.

data/lib/bolt/applicator.rb

@@ -34,6 +34,8 @@ module Bolt
           search_dirs << mod.plugins if mod.plugins?
           search_dirs << mod.pluginfacts if mod.pluginfacts?
           search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
           search_dirs
         end
       end
@@ -155,7 +157,6 @@ module Bolt
       end
 
       plan_vars = scope.to_hash(true, true)
-      %w[trusted server_facts facts].each { |k| plan_vars.delete(k) }
 
       targets = @inventory.get_targets(args[0])
 
@@ -182,7 +183,7 @@ module Bolt
                                                                        rich_data: true,
                                                                        symbol_as_string: true,
                                                                        type_by_reference: true,
-                                                                       local_reference: false)
+                                                                       local_reference: true)
 
       scope = {
         code_ast: ast,

data/lib/bolt/apply_inventory.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'bolt/boltdir'
+require 'bolt/project'
 require 'bolt/config'
 require 'bolt/error'
 

data/lib/bolt/apply_result.rb

@@ -57,7 +57,7 @@ module Bolt
           msg = "Report result contains an '_output' key. Catalog application may have printed extraneous output to stdout: #{result['_output']}"
           # rubocop:enable Layout/LineLength
         else
-          msg = "Report did not contain all expected keys missing: #{missing_keys.join(' ,')}"
+          msg = "Report did not contain all expected keys missing: #{missing_keys.join(', ')}"
         end
 
         { 'msg' => msg,

data/lib/bolt/apply_target.rb

@@ -3,7 +3,7 @@
 module Bolt
   class ApplyTarget
     ATTRIBUTES = %i[uri name target_alias config vars facts features
-                    plugin_hooks safe_name].freeze
+                    plugin_hooks resources safe_name].freeze
     COMPUTED = %i[host password port protocol user].freeze
 
     attr_reader(*ATTRIBUTES)
@@ -24,7 +24,8 @@ module Bolt
                                 facts = nil,
                                 vars = nil,
                                 features = nil,
-                                plugin_hooks = nil)
+                                plugin_hooks = nil,
+                                resources = nil)
       raise Bolt::Error.new("Target objects cannot be instantiated inside apply blocks", 'bolt/apply-error')
     end
     # rubocop:enable Lint/UnusedMethodArgument
@@ -57,6 +58,10 @@ module Bolt
       @user = Addressable::URI.unencode_component(uri_obj.user) || t_conf['user']
     end
 
+    def to_s
+      @safe_name
+    end
+
     def parse_uri(string)
       require 'addressable/uri'
       if string.nil?
@@ -73,5 +78,9 @@ module Bolt
     rescue Addressable::URI::InvalidURIError => e
       raise Bolt::ParseError, "Could not parse target URI: #{e.message}"
     end
+
+    def hash
+      @name.hash
+    end
   end
 end

data/lib/bolt/bolt_option_parser.rb

@@ -11,7 +11,7 @@ module Bolt
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
                 global_config_setters: %w[modulepath boltdir configfile],
-                transports: %w[transport connect-timeout tty],
+                transports: %w[transport connect-timeout tty ssh-command copy-command],
                 display: %w[format color verbose trace],
                 global: %w[help version debug] }.freeze
 
@@ -64,7 +64,7 @@ module Bolt
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters],
             banner: PLAN_CONVERT_HELP }
         when 'run'
-          { flags: ACTION_OPTS + %w[params compile-concurrency tmpdir],
+          { flags: ACTION_OPTS + %w[params compile-concurrency tmpdir hiera-config],
             banner: PLAN_RUN_HELP }
         when 'show'
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters] + %w[filter format],
@@ -112,7 +112,7 @@ module Bolt
       when 'secret'
         case action
         when 'createkeys'
-          { flags: OPTIONS[:global] + OPTIONS[:global_config_setters] + %w[plugin],
+          { flags: OPTIONS[:global] + OPTIONS[:global_config_setters] + %w[plugin force],
             banner: SECRET_CREATEKEYS_HELP }
         when 'decrypt'
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters] + %w[plugin],
@@ -594,6 +594,7 @@ module Bolt
     HELP
 
     attr_reader :warnings
+
     def initialize(options)
       super()
 
@@ -655,8 +656,8 @@ module Bolt
         @options[:password] = STDIN.noecho(&:gets).chomp
         STDERR.puts
       end
-      define('--private-key KEY', 'Private ssh key to authenticate with') do |key|
-        @options[:'private-key'] = key
+      define('--private-key KEY', 'Path to private ssh key to authenticate with') do |key|
+        @options[:'private-key'] = File.expand_path(key)
       end
       define('--[no-]host-key-check', 'Check host keys with SSH') do |host_key_check|
         @options[:'host-key-check'] = host_key_check
@@ -688,7 +689,7 @@ module Bolt
 
       separator "\nRUN CONTEXT OPTIONS"
       define('-c', '--concurrency CONCURRENCY', Integer,
-             'Maximum number of simultaneous connections (default: 100)') do |concurrency|
+             'Maximum number of simultaneous connections') do |concurrency|
         @options[:concurrency] = concurrency
       end
       define('--compile-concurrency CONCURRENCY', Integer,
@@ -716,6 +717,10 @@ module Bolt
              'Directory containing bolt.yaml will be used as the Boltdir.') do |path|
         @options[:configfile] = path
       end
+      define('--hiera-config FILEPATH',
+             'Specify where to load Hiera config from (default: ~/.puppetlabs/bolt/hiera.yaml)') do |path|
+        @options[:'hiera-config'] = File.expand_path(path)
+      end
       define('-i', '--inventoryfile FILEPATH',
              'Specify where to load inventory from (default: ~/.puppetlabs/bolt/inventory.yaml)') do |path|
         if ENV.include?(Bolt::Inventory::ENVIRONMENT_VAR)
@@ -737,6 +742,14 @@ module Bolt
              "Specify a default transport: #{TRANSPORTS.keys.join(', ')}") do |t|
         @options[:transport] = t
       end
+      define('--ssh-command EXEC', "Executable to use instead of the net-ssh ruby library. ",
+             "This option is experimental.") do |exec|
+        @options[:'ssh-command'] = exec
+      end
+      define('--copy-command EXEC', "Command to copy files to remote hosts if using external SSH. ",
+             "This option is experimental.") do |exec|
+        @options[:'copy-command'] = exec
+      end
       define('--connect-timeout TIMEOUT', Integer, 'Connection timeout (defaults vary)') do |timeout|
         @options[:'connect-timeout'] = timeout
       end
@@ -775,6 +788,9 @@ module Bolt
              'when initializing a project. Resolves and installs all dependencies.') do |modules|
         @options[:modules] = modules.split(',')
       end
+      define('--force', 'Overwrite existing key pairs') do |_force|
+        @options[:force] = true
+      end
 
       separator "\nGLOBAL OPTIONS"
       define('-h', '--help', 'Display help') do |_|

data/lib/bolt/cli.rb

@@ -114,18 +114,19 @@ module Bolt
       @config = if options[:configfile]
                   Bolt::Config.from_file(options[:configfile], options)
                 else
-                  boltdir = if options[:boltdir]
-                              Bolt::Boltdir.new(options[:boltdir])
+                  project = if options[:boltdir]
+                              Bolt::Project.new(options[:boltdir])
                             else
-                              Bolt::Boltdir.find_boltdir(Dir.pwd)
+                              Bolt::Project.find_boltdir(Dir.pwd)
                             end
-                  Bolt::Config.from_boltdir(boltdir, options)
+                  Bolt::Config.from_project(project, options)
                 end
 
       Bolt::Logger.configure(config.log, config.color)
 
-      # Logger must be configured before checking path case, otherwise warnings will not display
+      # Logger must be configured before checking path case and project file, otherwise warnings will not display
       @config.check_path_case('modulepath', @config.modulepath)
+      @config.project.check_deprecated_file
 
       # Log the file paths for loaded config files
       config_loaded
@@ -238,6 +239,12 @@ module Bolt
       if options[:subcommand] == 'command' && (!options[:object] || options[:object].empty?)
         raise Bolt::CLIError, "Must specify a command to run"
       end
+
+      if options[:subcommand] == 'secret' &&
+         (options[:action] == 'decrypt' || options[:action] == 'encrypt') &&
+         !options[:object]
+        raise Bolt::CLIError, "Must specify a value to #{options[:action]}"
+      end
     end
 
     def handle_parser_errors
@@ -251,13 +258,11 @@ module Bolt
     end
 
     def puppetdb_client
-      return @puppetdb_client if @puppetdb_client
-      puppetdb_config = Bolt::PuppetDB::Config.load_config(nil, config.puppetdb, config.boltdir.path)
-      @puppetdb_client = Bolt::PuppetDB::Client.new(puppetdb_config)
+      plugins.puppetdb_client
     end
 
     def plugins
-      @plugins ||= Bolt::Plugin.setup(config, pal, puppetdb_client, analytics)
+      @plugins ||= Bolt::Plugin.setup(config, pal, analytics)
     end
 
     def query_puppetdb_nodes(query)
@@ -314,7 +319,8 @@ module Bolt
 
       screen_view_fields = {
         output_format: config.format,
-        boltdir_type: config.boltdir.type
+        # For continuity
+        boltdir_type: config.project.type
       }
 
       # Only include target and inventory info for commands that take a targets
@@ -447,6 +453,7 @@ module Bolt
     def list_tasks
       tasks = pal.list_tasks
       tasks.select! { |task| task.first.include?(options[:filter]) } if options[:filter]
+      tasks.select! { |task| config.project.tasks.include?(task.first) } unless config.project.tasks.nil?
       outputter.print_tasks(tasks, pal.list_modulepath)
     end
 
@@ -457,6 +464,7 @@ module Bolt
     def list_plans
       plans = pal.list_plans
       plans.select! { |plan| plan.first.include?(options[:filter]) } if options[:filter]
+      plans.select! { |plan| config.project.plans.include?(plan.first) } unless config.project.plans.nil?
       outputter.print_plans(plans, pal.list_modulepath)
     end
 
@@ -567,10 +575,10 @@ module Bolt
     # Initializes a specified directory as a Bolt project and installs any modules
     # specified by the user, along with their dependencies
     def initialize_project
-      boltdir    = Pathname.new(File.expand_path(options[:object] || Dir.pwd))
-      config     = boltdir + 'bolt.yaml'
-      puppetfile = boltdir + 'Puppetfile'
-      modulepath = [boltdir + 'modules']
+      project    = Pathname.new(File.expand_path(options[:object] || Dir.pwd))
+      config     = project + 'bolt.yaml'
+      puppetfile = project + 'Puppetfile'
+      modulepath = [project + 'modules']
 
       # If modules were specified, first check if there is already a Puppetfile at the project
       # directory, erroring if there is. If there is no Puppetfile, generate the Puppetfile
@@ -590,20 +598,20 @@ module Bolt
       # Warn the user if the project directory already exists. We don't error here since users
       # might not have installed any modules yet.
       if config.exist?
-        @logger.warn "Found existing project directory at #{boltdir}"
+        @logger.warn "Found existing project directory at #{project}"
       end
 
       # Create the project directory
-      FileUtils.mkdir_p(boltdir)
+      FileUtils.mkdir_p(project)
 
-      # Bless the project directory as a boltdir
+      # Bless the project directory as a...wait for it...project
       if FileUtils.touch(config)
-        outputter.print_message "Successfully created Bolt project at #{boltdir}"
+        outputter.print_message "Successfully created Bolt project at #{project}"
       else
-        raise Bolt::FileError.new("Could not create Bolt project directory at #{boltdir}", nil)
+        raise Bolt::FileError.new("Could not create Bolt project directory at #{project}", nil)
       end
 
-      # Write the generated Puppetfile to the fancy new boltdir
+      # Write the generated Puppetfile to the fancy new project
       if puppetfile_specs
         File.write(puppetfile, puppetfile_specs.join("\n"))
         outputter.print_message "Successfully created Puppetfile at #{puppetfile}"
@@ -752,12 +760,14 @@ module Bolt
     end
 
     def pal
+      project = config.project.load_as_module? ? config.project : nil
       @pal ||= Bolt::PAL.new(config.modulepath,
                              config.hiera_config,
-                             config.boltdir.resource_types,
+                             config.project.resource_types,
                              config.compile_concurrency,
                              config.trusted_external,
-                             config.apply_settings)
+                             config.apply_settings,
+                             project)
     end
 
     def convert_plan(plan)
@@ -793,6 +803,20 @@ module Bolt
     end
 
     def bundled_content
+      # If the bundled content directory is empty, Bolt is likely installed as a gem.
+      if ENV['BOLT_GEM'].nil? && incomplete_install?
+        msg = <<~MSG.chomp
+          Bolt may be installed as a gem. To use Bolt reliably and with all of its
+          dependencies, uninstall the 'bolt' gem and install Bolt as a package:
+          https://puppet.com/docs/bolt/latest/bolt_installing.html
+
+          If you meant to install Bolt as a gem and want to disable this warning,
+          set the BOLT_GEM environment variable.
+        MSG
+
+        @logger.warn(msg)
+      end
+
       # We only need to enumerate bundled content when running a task or plan
       content = { 'Plan' => [],
                   'Task' => [],
@@ -816,5 +840,11 @@ module Bolt
       MSG
       @logger.debug(msg)
     end
+
+    # Gem installs include the aggregate, canary, and puppetdb_fact modules, while
+    # package installs include modules listed in the Bolt repo Puppetfile
+    def incomplete_install?
+      (Dir.children(Bolt::PAL::MODULES_PATH) - %w[aggregate canary puppetdb_fact]).empty?
+    end
   end
 end

data/lib/bolt/config.rb

@@ -3,7 +3,7 @@
 require 'etc'
 require 'logging'
 require 'pathname'
-require 'bolt/boltdir'
+require 'bolt/project'
 require 'bolt/logger'
 require 'bolt/util'
 # Transport config objects
@@ -23,7 +23,7 @@ module Bolt
   end
 
   class Config
-    attr_reader :config_files, :warnings, :data, :transports, :boltdir
+    attr_reader :config_files, :warnings, :data, :transports, :project
 
     TRANSPORT_CONFIG = {
       'ssh'    => Bolt::Config::Transport::SSH,
@@ -34,6 +34,8 @@ module Bolt
       'remote' => Bolt::Config::Transport::Remote
     }.freeze
 
+    # NOTE: All configuration options should have a corresponding schema property
+    #       in schemas/bolt-config.schema.json
     OPTIONS = {
       "apply_settings"           => "A map of Puppet settings to use when applying Puppet code",
       "color"                    => "Whether to use colored output when printing messages to the console.",
@@ -64,8 +66,8 @@ module Bolt
 
     DEFAULT_OPTIONS = {
       "color" => true,
-      "concurrency" => 100,
       "compile-concurrency" => "Number of cores",
+      "concurrency" => "100 or one-third of the ulimit, whichever is lower",
       "format" => "human",
       "hiera-config" => "Boltdir/hiera.yaml",
       "inventoryfile" => "Boltdir/inventory.yaml",
@@ -101,31 +103,33 @@ module Bolt
       "show_diff" => false
     }.freeze
 
+    DEFAULT_DEFAULT_CONCURRENCY = 100
+
     def self.default
-      new(Bolt::Boltdir.new('.'), {})
+      new(Bolt::Project.new('.'), {})
     end
 
-    def self.from_boltdir(boltdir, overrides = {})
+    def self.from_project(project, overrides = {})
       data = {
-        filepath: boltdir.config_file,
-        data: Bolt::Util.read_optional_yaml_hash(boltdir.config_file, 'config')
+        filepath: project.config_file,
+        data: Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
       }
 
       data = load_defaults.push(data).select { |config| config[:data]&.any? }
 
-      new(boltdir, data, overrides)
+      new(project, data, overrides)
     end
 
     def self.from_file(configfile, overrides = {})
-      boltdir = Bolt::Boltdir.new(Pathname.new(configfile).expand_path.dirname)
+      project = Bolt::Project.new(Pathname.new(configfile).expand_path.dirname)
 
       data = {
-        filepath: boltdir.config_file,
+        filepath: project.config_file,
         data: Bolt::Util.read_yaml_hash(configfile, 'config')
       }
       data = load_defaults.push(data).select { |config| config[:data]&.any? }
 
-      new(boltdir, data, overrides)
+      new(project, data, overrides)
     end
 
     def self.load_defaults
@@ -148,14 +152,14 @@ module Bolt
       confs
     end
 
-    def initialize(boltdir, config_data, overrides = {})
+    def initialize(project, config_data, overrides = {})
       unless config_data.is_a?(Array)
-        config_data = [{ filepath: boltdir.config_file, data: config_data }]
+        config_data = [{ filepath: project.config_file, data: config_data }]
       end
 
       @logger = Logging.logger[self]
       @warnings = []
-      @boltdir = boltdir
+      @project = project
       @transports = {}
       @config_files = []
 
@@ -163,7 +167,7 @@ module Bolt
         'apply_settings'      => {},
         'color'               => true,
         'compile-concurrency' => Etc.nprocessors,
-        'concurrency'         => 100,
+        'concurrency'         => default_concurrency,
         'format'              => 'human',
         'log'                 => { 'console' => {} },
         'plugin_hooks'        => {},
@@ -181,10 +185,22 @@ module Bolt
 
       override_data = normalize_overrides(overrides)
 
+      # If we need to lower concurrency and concurrency is not configured
+      ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
+      if default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+         !ld_concurrency &&
+         !override_data.key?('concurrency')
+        concurrency_warning = { option: 'concurrency',
+                                msg: "Concurrency will default to #{default_concurrency} because ulimit "\
+                                "is low: #{Etc.sysconf(Etc::SC_OPEN_MAX)}. Set concurrency with "\
+                                "'--concurrency', or set your ulimit with 'ulimit -n <limit>'" }
+        @warnings << concurrency_warning
+      end
+
       @data = merge_config_layers(default_data, *loaded_data, override_data)
 
       TRANSPORT_CONFIG.each do |transport, config|
-        @transports[transport] = config.new(@data.delete(transport), @boltdir.path)
+        @transports[transport] = config.new(@data.delete(transport), @project.path)
       end
 
       finalize_data
@@ -250,7 +266,7 @@ module Bolt
         @data['log'] = update_logs(@data['log'])
       end
 
-      # Expand paths relative to the Boltdir. Any settings that came from the
+      # Expand paths relative to the project. Any settings that came from the
       # CLI will already be absolute, so the expand will be skipped.
       if @data.key?('modulepath')
         moduledirs = if data['modulepath'].is_a?(String)
@@ -259,12 +275,12 @@ module Bolt
                        data['modulepath']
                      end
         @data['modulepath'] = moduledirs.map do |moduledir|
-          File.expand_path(moduledir, @boltdir.path)
+          File.expand_path(moduledir, @project.path)
         end
       end
 
       %w[hiera-config inventoryfile trusted-external-command].each do |opt|
-        @data[opt] = File.expand_path(@data[opt], @boltdir.path) if @data.key?(opt)
+        @data[opt] = File.expand_path(@data[opt], @project.path) if @data.key?(opt)
       end
 
       # Filter hashes to only include valid options
@@ -275,7 +291,7 @@ module Bolt
     private def normalize_log(target)
       return target if target == 'console'
       target = target[5..-1] if target.start_with?('file:')
-      'file:' + File.expand_path(target, @boltdir.path)
+      'file:' + File.expand_path(target, @project.path)
     end
 
     private def update_logs(logs)
@@ -310,10 +326,10 @@ module Bolt
         @warnings << { option: 'future', msg: msg }
       end
 
-      keys = OPTIONS.keys - %w[plugins plugin_hooks]
+      keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
-        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks]
+        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks puppetdb]
         raise Bolt::ValidationError,
               "Found unsupported key _plugin in config setting #{key}. Plugins are only available in "\
               "#{valid_keys.join(', ')}."
@@ -348,23 +364,23 @@ module Bolt
     end
 
     def default_inventoryfile
-      @boltdir.inventory_file
+      @project.inventory_file
     end
 
     def rerunfile
-      @boltdir.rerunfile
+      @project.rerunfile
     end
 
     def hiera_config
-      @data['hiera-config'] || @boltdir.hiera_config
+      @data['hiera-config'] || @project.hiera_config
     end
 
     def puppetfile
-      @puppetfile || @boltdir.puppetfile
+      @puppetfile || @project.puppetfile
     end
 
     def modulepath
-      @data['modulepath'] || @boltdir.modulepath
+      @data['modulepath'] || @project.modulepath
     end
 
     def modulepath=(value)
@@ -456,5 +472,19 @@ module Bolt
         l =~ /[A-Za-z]/ ? "[#{l.upcase}#{l.downcase}]" : l
       end.join
     end
+
+    # Etc::SC_OPEN_MAX is meaningless on windows, not defined in PE Jruby and not available
+    # on some platforms. This method holds the logic to decide whether or not to even consider it.
+    def sc_open_max_available?
+      !Bolt::Util.windows? && defined?(Etc::SC_OPEN_MAX) && Etc.sysconf(Etc::SC_OPEN_MAX)
+    end
+
+    def default_concurrency
+      @default_concurrency ||= if !sc_open_max_available? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300
+                                 DEFAULT_DEFAULT_CONCURRENCY
+                               else
+                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 3
+                               end
+    end
   end
 end