bolt 2.9.0 → 2.13.0

data/lib/bolt/analytics.rb

@@ -29,7 +29,7 @@ module Bolt
     def self.build_client
       logger = Logging.logger[self]
       begin
-        config_file = File.expand_path('~/.puppetlabs/bolt/analytics.yaml')
+        config_file = config_path(logger)
         config = load_config(config_file, logger)
       rescue ArgumentError
         config = { 'disabled' => true }
@@ -51,6 +51,25 @@ module Bolt
       NoopClient.new
     end
 
+    def self.config_path(logger)
+      path     = File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt', 'analytics.yaml'))
+      old_path = File.expand_path(File.join('~', '.puppetlabs', 'bolt', 'analytics.yaml'))
+
+      if File.exist?(path)
+        if File.exist?(old_path)
+          message = "Detected analytics configuration files at '#{old_path}' and '#{path}'. Loading "\
+                    "analytics configuration from '#{path}'."
+          logger.warn(message)
+        end
+
+        path
+      elsif File.exist?(old_path)
+        old_path
+      else
+        path
+      end
+    end
+
     def self.load_config(filename, logger)
       if File.exist?(filename)
         YAML.load_file(filename)
@@ -59,7 +78,7 @@ module Bolt
           logger.warn <<~ANALYTICS
             Bolt collects data about how you use it. You can opt out of providing this data.
 
-            To disable analytics data collection, add this line to ~/.puppetlabs/bolt/analytics.yaml :
+            To disable analytics data collection, add this line to ~/.puppetlabs/etc/bolt/analytics.yaml :
               disabled: true
 
             Read more about what data Bolt collects and why here:

data/lib/bolt/applicator.rb

@@ -14,7 +14,8 @@ require 'open3'
 
 module Bolt
   class Applicator
-    def initialize(inventory, executor, modulepath, plugin_dirs, pdb_client, hiera_config, max_compiles, apply_settings)
+    def initialize(inventory, executor, modulepath, plugin_dirs, project,
+                   pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
 
@@ -22,6 +23,7 @@ module Bolt
       @executor = executor
       @modulepath = modulepath
       @plugin_dirs = plugin_dirs
+      @project = project
       @pdb_client = pdb_client
       @hiera_config = hiera_config ? validate_hiera_config(hiera_config) : nil
       @apply_settings = apply_settings || {}
@@ -34,6 +36,8 @@ module Bolt
           search_dirs << mod.plugins if mod.plugins?
           search_dirs << mod.pluginfacts if mod.pluginfacts?
           search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
           search_dirs
         end
       end
@@ -181,11 +185,12 @@ module Bolt
                                                                        rich_data: true,
                                                                        symbol_as_string: true,
                                                                        type_by_reference: true,
-                                                                       local_reference: false)
+                                                                       local_reference: true)
 
       scope = {
         code_ast: ast,
         modulepath: @modulepath,
+        project: @project.to_h,
         pdb_config: @pdb_client.config.to_hash,
         hiera_config: @hiera_config,
         plan_vars: plan_vars,

data/lib/bolt/apply_result.rb

@@ -57,7 +57,7 @@ module Bolt
           msg = "Report result contains an '_output' key. Catalog application may have printed extraneous output to stdout: #{result['_output']}"
           # rubocop:enable Layout/LineLength
         else
-          msg = "Report did not contain all expected keys missing: #{missing_keys.join(' ,')}"
+          msg = "Report did not contain all expected keys missing: #{missing_keys.join(', ')}"
         end
 
         { 'msg' => msg,

data/lib/bolt/apply_target.rb

@@ -3,7 +3,7 @@
 module Bolt
   class ApplyTarget
     ATTRIBUTES = %i[uri name target_alias config vars facts features
-                    plugin_hooks safe_name].freeze
+                    plugin_hooks resources safe_name].freeze
     COMPUTED = %i[host password port protocol user].freeze
 
     attr_reader(*ATTRIBUTES)
@@ -24,7 +24,8 @@ module Bolt
                                 facts = nil,
                                 vars = nil,
                                 features = nil,
-                                plugin_hooks = nil)
+                                plugin_hooks = nil,
+                                resources = nil)
       raise Bolt::Error.new("Target objects cannot be instantiated inside apply blocks", 'bolt/apply-error')
     end
     # rubocop:enable Lint/UnusedMethodArgument

data/lib/bolt/bolt_option_parser.rb

@@ -11,7 +11,7 @@ module Bolt
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
                 global_config_setters: %w[modulepath boltdir configfile],
-                transports: %w[transport connect-timeout tty],
+                transports: %w[transport connect-timeout tty ssh-command copy-command],
                 display: %w[format color verbose trace],
                 global: %w[help version debug] }.freeze
 
@@ -20,7 +20,7 @@ module Bolt
     def get_help_text(subcommand, action = nil)
       case subcommand
       when 'apply'
-        { flags: ACTION_OPTS + %w[noop execute compile-concurrency],
+        { flags: ACTION_OPTS + %w[noop execute compile-concurrency hiera-config],
           banner: APPLY_HELP }
       when 'command'
         case action
@@ -172,13 +172,14 @@ module Bolt
           apply
 
       USAGE
-          bolt apply <manifest.pp> [options]
+          bolt apply [manifest.pp] [options]
 
       DESCRIPTION
           Apply Puppet manifest code on the specified targets.
 
       EXAMPLES
-          bolt apply manifest.pp --targets target1,target2
+          bolt apply manifest.pp -t target
+          bolt apply -e "file { '/etc/puppetlabs': ensure => present }" -t target
     HELP
 
     COMMAND_HELP = <<~HELP
@@ -594,6 +595,7 @@ module Bolt
     HELP
 
     attr_reader :warnings
+
     def initialize(options)
       super()
 
@@ -655,8 +657,8 @@ module Bolt
         @options[:password] = STDIN.noecho(&:gets).chomp
         STDERR.puts
       end
-      define('--private-key KEY', 'Private ssh key to authenticate with') do |key|
-        @options[:'private-key'] = key
+      define('--private-key KEY', 'Path to private ssh key to authenticate with') do |key|
+        @options[:'private-key'] = File.expand_path(key)
       end
       define('--[no-]host-key-check', 'Check host keys with SSH') do |host_key_check|
         @options[:'host-key-check'] = host_key_check
@@ -688,7 +690,7 @@ module Bolt
 
       separator "\nRUN CONTEXT OPTIONS"
       define('-c', '--concurrency CONCURRENCY', Integer,
-             'Maximum number of simultaneous connections (default: 100)') do |concurrency|
+             'Maximum number of simultaneous connections') do |concurrency|
         @options[:concurrency] = concurrency
       end
       define('--compile-concurrency CONCURRENCY', Integer,
@@ -718,7 +720,7 @@ module Bolt
       end
       define('--hiera-config FILEPATH',
              'Specify where to load Hiera config from (default: ~/.puppetlabs/bolt/hiera.yaml)') do |path|
-        @options[:'hiera-config'] = path
+        @options[:'hiera-config'] = File.expand_path(path)
       end
       define('-i', '--inventoryfile FILEPATH',
              'Specify where to load inventory from (default: ~/.puppetlabs/bolt/inventory.yaml)') do |path|
@@ -741,6 +743,14 @@ module Bolt
              "Specify a default transport: #{TRANSPORTS.keys.join(', ')}") do |t|
         @options[:transport] = t
       end
+      define('--ssh-command EXEC', "Executable to use instead of the net-ssh ruby library. ",
+             "This option is experimental.") do |exec|
+        @options[:'ssh-command'] = exec
+      end
+      define('--copy-command EXEC', "Command to copy files to remote hosts if using external SSH. ",
+             "This option is experimental.") do |exec|
+        @options[:'copy-command'] = exec
+      end
       define('--connect-timeout TIMEOUT', Integer, 'Connection timeout (defaults vary)') do |timeout|
         @options[:'connect-timeout'] = timeout
       end

data/lib/bolt/catalog.rb

@@ -58,6 +58,8 @@ module Bolt
       target = request['target']
       pdb_client = Bolt::PuppetDB::Client.new(Bolt::PuppetDB::Config.new(request['pdb_config']))
       options = request['puppet_config'] || {}
+      project = request['project'] || {}
+      bolt_project = Struct.new(:name, :path).new(project['name'], project['path']) unless project.empty?
       with_puppet_settings(request['hiera_config']) do
         Puppet[:rich_data] = true
         Puppet[:node_name_value] = target['name']
@@ -67,7 +69,8 @@ module Bolt
         Puppet::Pal.in_tmp_environment('bolt_catalog', env_conf) do |pal|
           inv = Bolt::ApplyInventory.new(request['config'])
           Puppet.override(bolt_pdb_client: pdb_client,
-                          bolt_inventory: inv) do
+                          bolt_inventory: inv,
+                          bolt_project: bolt_project) do
             Puppet.lookup(:pal_current_node).trusted_data = target['trusted']
             pal.with_catalog_compiler do |compiler|
               # Deserializing needs to happen inside the catalog compiler so

data/lib/bolt/cli.rb

@@ -124,8 +124,9 @@ module Bolt
 
       Bolt::Logger.configure(config.log, config.color)
 
-      # Logger must be configured before checking path case, otherwise warnings will not display
+      # Logger must be configured before checking path case and project file, otherwise warnings will not display
       @config.check_path_case('modulepath', @config.modulepath)
+      @config.project.check_deprecated_file
 
       # Log the file paths for loaded config files
       config_loaded
@@ -257,13 +258,11 @@ module Bolt
     end
 
     def puppetdb_client
-      return @puppetdb_client if @puppetdb_client
-      puppetdb_config = Bolt::PuppetDB::Config.load_config(nil, config.puppetdb, config.project.path)
-      @puppetdb_client = Bolt::PuppetDB::Client.new(puppetdb_config)
+      plugins.puppetdb_client
     end
 
     def plugins
-      @plugins ||= Bolt::Plugin.setup(config, pal, puppetdb_client, analytics)
+      @plugins ||= Bolt::Plugin.setup(config, pal, analytics)
     end
 
     def query_puppetdb_nodes(query)
@@ -538,6 +537,17 @@ module Bolt
       Puppet[:tasks] = false
       ast = pal.parse_manifest(code, filename)
 
+      if defined?(ast.body) &&
+         (ast.body.is_a?(Puppet::Pops::Model::HostClassDefinition) ||
+         ast.body.is_a?(Puppet::Pops::Model::ResourceTypeDefinition))
+        message = "Manifest only contains definitions and will result in no changes on the targets. "\
+                  "Definitions must be declared for their resources to be applied. You can read more "\
+                  "about defining and declaring classes and types in the Puppet documentation at "\
+                  "https://puppet.com/docs/puppet/latest/lang_classes.html and "\
+                  "https://puppet.com/docs/puppet/latest/lang_defined_types.html"
+        @logger.warn(message)
+      end
+
       executor = Bolt::Executor.new(config.concurrency, analytics, noop)
       executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
       executor.subscribe(log_outputter)

data/lib/bolt/config.rb

@@ -47,9 +47,7 @@ module Bolt
                                     "targets on the command line and from plans.",
       "log"                      => "The configuration of the logfile output. Configuration can be set for "\
                                     "`console` and the path to a log file, such as `~/.puppetlabs/bolt/debug.log`.",
-      "modulepath"               => "The module path for loading tasks and plan code. This is either an array "\
-                                    "of directories or a string containing a list of directories separated by the "\
-                                    "OS-specific PATH separator.",
+      "modulepath"               => "An array of directories that Bolt loads content (e.g. plans and tasks) from.",
       "plugin_hooks"             => "Which plugins a specific hook should use.",
       "plugins"                  => "A map of plugins and their configuration data.",
       "puppetdb"                 => "A map containing options for configuring the Bolt PuppetDB client.",
@@ -66,8 +64,8 @@ module Bolt
 
     DEFAULT_OPTIONS = {
       "color" => true,
-      "concurrency" => 100,
       "compile-concurrency" => "Number of cores",
+      "concurrency" => "100 or one-third of the ulimit, whichever is lower",
       "format" => "human",
       "hiera-config" => "Boltdir/hiera.yaml",
       "inventoryfile" => "Boltdir/inventory.yaml",
@@ -103,6 +101,8 @@ module Bolt
       "show_diff" => false
     }.freeze
 
+    DEFAULT_DEFAULT_CONCURRENCY = 100
+
     def self.default
       new(Bolt::Project.new('.'), {})
     end
@@ -113,7 +113,7 @@ module Bolt
         data: Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
       }
 
-      data = load_defaults.push(data).select { |config| config[:data]&.any? }
+      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
 
       new(project, data, overrides)
     end
@@ -125,27 +125,29 @@ module Bolt
         filepath: project.config_file,
         data: Bolt::Util.read_yaml_hash(configfile, 'config')
       }
-      data = load_defaults.push(data).select { |config| config[:data]&.any? }
+      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
 
       new(project, data, overrides)
     end
 
-    def self.load_defaults
+    def self.load_defaults(project)
       # Lazy-load expensive gem code
       require 'win32/dir' if Bolt::Util.windows?
 
-      system_path = if Bolt::Util.windows?
-                      Pathname.new(File.join(Dir::COMMON_APPDATA, 'PuppetLabs', 'bolt', 'etc', 'bolt.yaml'))
-                    else
-                      Pathname.new(File.join('/etc', 'puppetlabs', 'bolt', 'bolt.yaml'))
-                    end
+      # Don't load /etc/puppetlabs/bolt/bolt.yaml twice
+      confs = if project.path == Bolt::Project.system_path
+                []
+              else
+                system_path = Pathname.new(File.join(Bolt::Project.system_path, 'bolt.yaml'))
+                [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
+              end
+
       user_path = begin
                     Pathname.new(File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt', 'bolt.yaml')))
                   rescue ArgumentError
                     nil
                   end
 
-      confs = [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
       confs << { filepath: user_path, data: Bolt::Util.read_optional_yaml_hash(user_path, 'config') } if user_path
       confs
     end
@@ -165,7 +167,7 @@ module Bolt
         'apply_settings'      => {},
         'color'               => true,
         'compile-concurrency' => Etc.nprocessors,
-        'concurrency'         => 100,
+        'concurrency'         => default_concurrency,
         'format'              => 'human',
         'log'                 => { 'console' => {} },
         'plugin_hooks'        => {},
@@ -183,6 +185,18 @@ module Bolt
 
       override_data = normalize_overrides(overrides)
 
+      # If we need to lower concurrency and concurrency is not configured
+      ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
+      if default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+         !ld_concurrency &&
+         !override_data.key?('concurrency')
+        concurrency_warning = { option: 'concurrency',
+                                msg: "Concurrency will default to #{default_concurrency} because ulimit "\
+                                "is low: #{Etc.sysconf(Etc::SC_OPEN_MAX)}. Set concurrency with "\
+                                "'--concurrency', or set your ulimit with 'ulimit -n <limit>'" }
+        @warnings << concurrency_warning
+      end
+
       @data = merge_config_layers(default_data, *loaded_data, override_data)
 
       TRANSPORT_CONFIG.each do |transport, config|
@@ -312,10 +326,10 @@ module Bolt
         @warnings << { option: 'future', msg: msg }
       end
 
-      keys = OPTIONS.keys - %w[plugins plugin_hooks]
+      keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
-        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks]
+        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks puppetdb]
         raise Bolt::ValidationError,
               "Found unsupported key _plugin in config setting #{key}. Plugins are only available in "\
               "#{valid_keys.join(', ')}."
@@ -458,5 +472,19 @@ module Bolt
         l =~ /[A-Za-z]/ ? "[#{l.upcase}#{l.downcase}]" : l
       end.join
     end
+
+    # Etc::SC_OPEN_MAX is meaningless on windows, not defined in PE Jruby and not available
+    # on some platforms. This method holds the logic to decide whether or not to even consider it.
+    def sc_open_max_available?
+      !Bolt::Util.windows? && defined?(Etc::SC_OPEN_MAX) && Etc.sysconf(Etc::SC_OPEN_MAX)
+    end
+
+    def default_concurrency
+      @default_concurrency ||= if !sc_open_max_available? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300
+                                 DEFAULT_DEFAULT_CONCURRENCY
+                               else
+                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 3
+                               end
+    end
   end
 end

data/lib/bolt/config/transport/ssh.rb

@@ -13,14 +13,20 @@ module Bolt
         #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "cleanup"            => { type: TrueClass,
+                                    external: true,
                                     desc: "Whether to clean up temporary files created on targets." },
           "connect-timeout"    => { type: Integer,
                                     desc: "How long to wait when establishing connections." },
+          "copy-command"       => { external: true,
+                                    desc: "Command to use when copying files using ssh-command. "\
+                                          "Bolt runs `<copy-command> <src> <dest>`. **This option is experimental.**" },
           "disconnect-timeout" => { type: Integer,
                                     desc: "How long to wait before force-closing a connection." },
           "host"               => { type: String,
+                                    external: true,
                                     desc: "Host name." },
           "host-key-check"     => { type: TrueClass,
+                                    external: true,
                                     desc: "Whether to perform host key validation when connecting." },
           "extensions"         => { type: Array,
                                     desc: "List of file extensions that are accepted for scripts or tasks on Windows. "\
@@ -29,6 +35,7 @@ module Bolt
                                          "a `.py` script runs with `python.exe`. The extensions `.ps1`, `.rb`, and "\
                                          "`.pp` are always allowed and run via hard-coded executables." },
           "interpreters"       => { type: Hash,
+                                    external: true,
                                     desc: "A map of an extension name to the absolute path of an executable, "\
                                           "enabling you to override the shebang defined in a task executable. The "\
                                           "extension can optionally be specified with the `.` character (`.py` and "\
@@ -44,26 +51,36 @@ module Bolt
           "password"           => { type: String,
                                     desc: "Login password." },
           "port"               => { type: Integer,
+                                    external: true,
                                     desc: "Connection port." },
-          "private-key"        => { desc: "Either the path to the private key file to use for authentication, or a "\
+          "private-key"        => { external: true,
+                                    desc: "Either the path to the private key file to use for authentication, or a "\
                                           "hash with the key `key-data` and the contents of the private key." },
           "proxyjump"          => { type: String,
                                     desc: "A jump host to proxy connections through, and an optional user to "\
                                           "connect with." },
           "run-as"             => { type: String,
+                                    external: true,
                                     desc: "A different user to run commands as after login." },
           "run-as-command"     => { type: Array,
+                                    external: true,
                                     desc: "The command to elevate permissions. Bolt appends the user and command "\
                                           "strings to the configured `run-as-command` before running it on the "\
                                           "target. This command must not require an interactive password prompt, "\
                                           "and the `sudo-password` option is ignored when `run-as-command` is "\
                                           "specified. The `run-as-command` must be specified as an array." },
           "script-dir"         => { type: String,
+                                    external: true,
                                     desc: "The subdirectory of the tmpdir to use in place of a randomized "\
                                           "subdirectory for uploading and executing temporary files on the "\
                                           "target. It's expected that this directory already exists as a subdir "\
                                           "of tmpdir, which is either configured or defaults to `/tmp`." },
+          "ssh-command"        => { external: true,
+                                    desc: "Command and flags to use when SSHing. This enables the external "\
+                                          "SSH transport which shells out to the specified command. "\
+                                          "**This option is experimental.**" },
           "sudo-executable"    => { type: String,
+                                    external: true,
                                     desc: "The executable to use when escalating to the configured `run-as` "\
                                           "user. This is useful when you want to escalate using the configured "\
                                           "`sudo-password`, since `run-as-command` does not use `sudo-password` "\
@@ -71,14 +88,17 @@ module Bolt
                                           "`<sudo-executable> -S -u <user> -p custom_bolt_prompt <command>`. "\
                                           "**This option is experimental.**" },
           "sudo-password"      => { type: String,
+                                    external: true,
                                     desc: "Password to use when changing users via `run-as`." },
           "tmpdir"             => { type: String,
+                                    external: true,
                                     desc: "The directory to upload and execute temporary files on the target." },
           "tty"                => { type: TrueClass,
                                     desc: "Request a pseudo tty for the session. This option is generally "\
                                           "only used in conjunction with the `run-as` option when the sudoers "\
                                           "policy requires a `tty`." },
           "user"               => { type: String,
+                                    external: true,
                                     desc: "Login user." }
         }.freeze
 
@@ -103,6 +123,13 @@ module Bolt
 
             if key_opt.instance_of?(String)
               @config['private-key'] = File.expand_path(key_opt, @project)
+
+              # We have an explicit test for this to only warn if using net-ssh transport
+              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['ssh-command']
+            end
+
+            if key_opt.instance_of?(Hash) && @config['ssh-command']
+              raise Bolt::ValidationError, 'private-key must be a filepath when using ssh-command'
             end
           end
 
@@ -130,6 +157,25 @@ module Bolt
               end
             end
           end
+
+          if @config['ssh-command'] && !@config['load-config']
+            msg = 'Cannot use external SSH transport with load-config set to false'
+            raise Bolt::ValidationError, msg
+          end
+
+          if (ssh_cmd = @config['ssh-command'])
+            unless ssh_cmd.is_a?(String) || ssh_cmd.is_a?(Array)
+              raise Bolt::ValidationError,
+                    "ssh-command must be a String or Array, received #{ssh_cmd.class} #{ssh_cmd.inspect}"
+            end
+          end
+
+          if (copy_cmd = @config['copy-command'])
+            unless copy_cmd.is_a?(String) || copy_cmd.is_a?(Array)
+              raise Bolt::ValidationError,
+                    "copy-command must be a String or Array, received #{copy_cmd.class} #{copy_cmd.inspect}"
+            end
+          end
         end
       end
     end