bolt 2.8.0 → 2.12.0

data/lib/bolt/apply_target.rb

@@ -3,7 +3,7 @@
 module Bolt
   class ApplyTarget
     ATTRIBUTES = %i[uri name target_alias config vars facts features
-                    plugin_hooks safe_name].freeze
+                    plugin_hooks resources safe_name].freeze
     COMPUTED = %i[host password port protocol user].freeze
 
     attr_reader(*ATTRIBUTES)
@@ -24,7 +24,8 @@ module Bolt
                                 facts = nil,
                                 vars = nil,
                                 features = nil,
-                                plugin_hooks = nil)
+                                plugin_hooks = nil,
+                                resources = nil)
       raise Bolt::Error.new("Target objects cannot be instantiated inside apply blocks", 'bolt/apply-error')
     end
     # rubocop:enable Lint/UnusedMethodArgument

data/lib/bolt/bolt_option_parser.rb

@@ -11,7 +11,7 @@ module Bolt
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
                 global_config_setters: %w[modulepath boltdir configfile],
-                transports: %w[transport connect-timeout tty],
+                transports: %w[transport connect-timeout tty ssh-command copy-command],
                 display: %w[format color verbose trace],
                 global: %w[help version debug] }.freeze
 
@@ -20,7 +20,7 @@ module Bolt
     def get_help_text(subcommand, action = nil)
       case subcommand
       when 'apply'
-        { flags: ACTION_OPTS + %w[noop execute compile-concurrency],
+        { flags: ACTION_OPTS + %w[noop execute compile-concurrency hiera-config],
           banner: APPLY_HELP }
       when 'command'
         case action
@@ -172,13 +172,14 @@ module Bolt
           apply
 
       USAGE
-          bolt apply <manifest.pp> [options]
+          bolt apply [manifest.pp] [options]
 
       DESCRIPTION
           Apply Puppet manifest code on the specified targets.
 
       EXAMPLES
-          bolt apply manifest.pp --targets target1,target2
+          bolt apply manifest.pp -t target
+          bolt apply -e "file { '/etc/puppetlabs': ensure => present }" -t target
     HELP
 
     COMMAND_HELP = <<~HELP
@@ -594,6 +595,7 @@ module Bolt
     HELP
 
     attr_reader :warnings
+
     def initialize(options)
       super()
 
@@ -655,8 +657,8 @@ module Bolt
         @options[:password] = STDIN.noecho(&:gets).chomp
         STDERR.puts
       end
-      define('--private-key KEY', 'Private ssh key to authenticate with') do |key|
-        @options[:'private-key'] = key
+      define('--private-key KEY', 'Path to private ssh key to authenticate with') do |key|
+        @options[:'private-key'] = File.expand_path(key)
       end
       define('--[no-]host-key-check', 'Check host keys with SSH') do |host_key_check|
         @options[:'host-key-check'] = host_key_check
@@ -688,7 +690,7 @@ module Bolt
 
       separator "\nRUN CONTEXT OPTIONS"
       define('-c', '--concurrency CONCURRENCY', Integer,
-             'Maximum number of simultaneous connections (default: 100)') do |concurrency|
+             'Maximum number of simultaneous connections') do |concurrency|
         @options[:concurrency] = concurrency
       end
       define('--compile-concurrency CONCURRENCY', Integer,
@@ -718,7 +720,7 @@ module Bolt
       end
       define('--hiera-config FILEPATH',
              'Specify where to load Hiera config from (default: ~/.puppetlabs/bolt/hiera.yaml)') do |path|
-        @options[:'hiera-config'] = path
+        @options[:'hiera-config'] = File.expand_path(path)
       end
       define('-i', '--inventoryfile FILEPATH',
              'Specify where to load inventory from (default: ~/.puppetlabs/bolt/inventory.yaml)') do |path|
@@ -741,6 +743,14 @@ module Bolt
              "Specify a default transport: #{TRANSPORTS.keys.join(', ')}") do |t|
         @options[:transport] = t
       end
+      define('--ssh-command EXEC', "Executable to use instead of the net-ssh ruby library. ",
+             "This option is experimental.") do |exec|
+        @options[:'ssh-command'] = exec
+      end
+      define('--copy-command EXEC', "Command to copy files to remote hosts if using external SSH. ",
+             "This option is experimental.") do |exec|
+        @options[:'copy-command'] = exec
+      end
       define('--connect-timeout TIMEOUT', Integer, 'Connection timeout (defaults vary)') do |timeout|
         @options[:'connect-timeout'] = timeout
       end

data/lib/bolt/cli.rb

@@ -124,8 +124,9 @@ module Bolt
 
       Bolt::Logger.configure(config.log, config.color)
 
-      # Logger must be configured before checking path case, otherwise warnings will not display
+      # Logger must be configured before checking path case and project file, otherwise warnings will not display
       @config.check_path_case('modulepath', @config.modulepath)
+      @config.project.check_deprecated_file
 
       # Log the file paths for loaded config files
       config_loaded
@@ -257,13 +258,11 @@ module Bolt
     end
 
     def puppetdb_client
-      return @puppetdb_client if @puppetdb_client
-      puppetdb_config = Bolt::PuppetDB::Config.load_config(nil, config.puppetdb, config.project.path)
-      @puppetdb_client = Bolt::PuppetDB::Client.new(puppetdb_config)
+      plugins.puppetdb_client
     end
 
     def plugins
-      @plugins ||= Bolt::Plugin.setup(config, pal, puppetdb_client, analytics)
+      @plugins ||= Bolt::Plugin.setup(config, pal, analytics)
     end
 
     def query_puppetdb_nodes(query)
@@ -538,6 +537,17 @@ module Bolt
       Puppet[:tasks] = false
       ast = pal.parse_manifest(code, filename)
 
+      if defined?(ast.body) &&
+         (ast.body.is_a?(Puppet::Pops::Model::HostClassDefinition) ||
+         ast.body.is_a?(Puppet::Pops::Model::ResourceTypeDefinition))
+        message = "Manifest only contains definitions and will result in no changes on the targets. "\
+                  "Definitions must be declared for their resources to be applied. You can read more "\
+                  "about defining and declaring classes and types in the Puppet documentation at "\
+                  "https://puppet.com/docs/puppet/latest/lang_classes.html and "\
+                  "https://puppet.com/docs/puppet/latest/lang_defined_types.html"
+        @logger.warn(message)
+      end
+
       executor = Bolt::Executor.new(config.concurrency, analytics, noop)
       executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
       executor.subscribe(log_outputter)
@@ -804,6 +814,20 @@ module Bolt
     end
 
     def bundled_content
+      # If the bundled content directory is empty, Bolt is likely installed as a gem.
+      if ENV['BOLT_GEM'].nil? && incomplete_install?
+        msg = <<~MSG.chomp
+          Bolt may be installed as a gem. To use Bolt reliably and with all of its
+          dependencies, uninstall the 'bolt' gem and install Bolt as a package:
+          https://puppet.com/docs/bolt/latest/bolt_installing.html
+
+          If you meant to install Bolt as a gem and want to disable this warning,
+          set the BOLT_GEM environment variable.
+        MSG
+
+        @logger.warn(msg)
+      end
+
       # We only need to enumerate bundled content when running a task or plan
       content = { 'Plan' => [],
                   'Task' => [],
@@ -827,5 +851,11 @@ module Bolt
       MSG
       @logger.debug(msg)
     end
+
+    # Gem installs include the aggregate, canary, and puppetdb_fact modules, while
+    # package installs include modules listed in the Bolt repo Puppetfile
+    def incomplete_install?
+      (Dir.children(Bolt::PAL::MODULES_PATH) - %w[aggregate canary puppetdb_fact]).empty?
+    end
   end
 end

data/lib/bolt/config.rb

@@ -34,6 +34,8 @@ module Bolt
       'remote' => Bolt::Config::Transport::Remote
     }.freeze
 
+    # NOTE: All configuration options should have a corresponding schema property
+    #       in schemas/bolt-config.schema.json
     OPTIONS = {
       "apply_settings"           => "A map of Puppet settings to use when applying Puppet code",
       "color"                    => "Whether to use colored output when printing messages to the console.",
@@ -64,8 +66,8 @@ module Bolt
 
     DEFAULT_OPTIONS = {
       "color" => true,
-      "concurrency" => 100,
       "compile-concurrency" => "Number of cores",
+      "concurrency" => "100 or one-third of the ulimit, whichever is lower",
       "format" => "human",
       "hiera-config" => "Boltdir/hiera.yaml",
       "inventoryfile" => "Boltdir/inventory.yaml",
@@ -101,6 +103,8 @@ module Bolt
       "show_diff" => false
     }.freeze
 
+    DEFAULT_DEFAULT_CONCURRENCY = 100
+
     def self.default
       new(Bolt::Project.new('.'), {})
     end
@@ -111,7 +115,7 @@ module Bolt
         data: Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
       }
 
-      data = load_defaults.push(data).select { |config| config[:data]&.any? }
+      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
 
       new(project, data, overrides)
     end
@@ -123,27 +127,29 @@ module Bolt
         filepath: project.config_file,
         data: Bolt::Util.read_yaml_hash(configfile, 'config')
       }
-      data = load_defaults.push(data).select { |config| config[:data]&.any? }
+      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
 
       new(project, data, overrides)
     end
 
-    def self.load_defaults
+    def self.load_defaults(project)
       # Lazy-load expensive gem code
       require 'win32/dir' if Bolt::Util.windows?
 
-      system_path = if Bolt::Util.windows?
-                      Pathname.new(File.join(Dir::COMMON_APPDATA, 'PuppetLabs', 'bolt', 'etc', 'bolt.yaml'))
-                    else
-                      Pathname.new(File.join('/etc', 'puppetlabs', 'bolt', 'bolt.yaml'))
-                    end
+      # Don't load /etc/puppetlabs/bolt/bolt.yaml twice
+      confs = if project.path == Bolt::Project.system_path
+                []
+              else
+                system_path = Pathname.new(File.join(Bolt::Project.system_path, 'bolt.yaml'))
+                [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
+              end
+
       user_path = begin
                     Pathname.new(File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt', 'bolt.yaml')))
                   rescue ArgumentError
                     nil
                   end
 
-      confs = [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
       confs << { filepath: user_path, data: Bolt::Util.read_optional_yaml_hash(user_path, 'config') } if user_path
       confs
     end
@@ -163,7 +169,7 @@ module Bolt
         'apply_settings'      => {},
         'color'               => true,
         'compile-concurrency' => Etc.nprocessors,
-        'concurrency'         => 100,
+        'concurrency'         => default_concurrency,
         'format'              => 'human',
         'log'                 => { 'console' => {} },
         'plugin_hooks'        => {},
@@ -181,6 +187,18 @@ module Bolt
 
       override_data = normalize_overrides(overrides)
 
+      # If we need to lower concurrency and concurrency is not configured
+      ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
+      if default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+         !ld_concurrency &&
+         !override_data.key?('concurrency')
+        concurrency_warning = { option: 'concurrency',
+                                msg: "Concurrency will default to #{default_concurrency} because ulimit "\
+                                "is low: #{Etc.sysconf(Etc::SC_OPEN_MAX)}. Set concurrency with "\
+                                "'--concurrency', or set your ulimit with 'ulimit -n <limit>'" }
+        @warnings << concurrency_warning
+      end
+
       @data = merge_config_layers(default_data, *loaded_data, override_data)
 
       TRANSPORT_CONFIG.each do |transport, config|
@@ -310,10 +328,10 @@ module Bolt
         @warnings << { option: 'future', msg: msg }
       end
 
-      keys = OPTIONS.keys - %w[plugins plugin_hooks]
+      keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
-        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks]
+        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks puppetdb]
         raise Bolt::ValidationError,
               "Found unsupported key _plugin in config setting #{key}. Plugins are only available in "\
               "#{valid_keys.join(', ')}."
@@ -456,5 +474,19 @@ module Bolt
         l =~ /[A-Za-z]/ ? "[#{l.upcase}#{l.downcase}]" : l
       end.join
     end
+
+    # Etc::SC_OPEN_MAX is meaningless on windows, not defined in PE Jruby and not available
+    # on some platforms. This method holds the logic to decide whether or not to even consider it.
+    def sc_open_max_available?
+      !Bolt::Util.windows? && defined?(Etc::SC_OPEN_MAX) && Etc.sysconf(Etc::SC_OPEN_MAX)
+    end
+
+    def default_concurrency
+      @default_concurrency ||= if !sc_open_max_available? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300
+                                 DEFAULT_DEFAULT_CONCURRENCY
+                               else
+                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 3
+                               end
+    end
   end
 end

data/lib/bolt/config/transport/docker.rb

@@ -7,6 +7,8 @@ module Bolt
   class Config
     module Transport
       class Docker < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "cleanup"       => { type: TrueClass,
                                desc: "Whether to clean up temporary files created on targets." },

data/lib/bolt/config/transport/local.rb

@@ -7,6 +7,8 @@ module Bolt
   class Config
     module Transport
       class Local < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "cleanup"         => { type: TrueClass,
                                  desc: "Whether to clean up temporary files created on targets." },

data/lib/bolt/config/transport/orch.rb

@@ -7,6 +7,8 @@ module Bolt
   class Config
     module Transport
       class Orch < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "cacert"            => { type: String,
                                    desc: "The path to the CA certificate." },

data/lib/bolt/config/transport/remote.rb

@@ -7,6 +7,8 @@ module Bolt
   class Config
     module Transport
       class Remote < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "run-on" => { type: String,
                         desc: "The proxy target that the task executes on." }

data/lib/bolt/config/transport/ssh.rb

@@ -8,16 +8,25 @@ module Bolt
     module Transport
       class SSH < Base
         LOGIN_SHELLS = %w[sh bash zsh dash ksh powershell].freeze
+
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "cleanup"            => { type: TrueClass,
+                                    external: true,
                                     desc: "Whether to clean up temporary files created on targets." },
           "connect-timeout"    => { type: Integer,
                                     desc: "How long to wait when establishing connections." },
+          "copy-command"       => { external: true,
+                                    desc: "Command to use when copying files using ssh-command. "\
+                                          "Bolt runs `<copy-command> <src> <dest>`. **This option is experimental.**" },
           "disconnect-timeout" => { type: Integer,
                                     desc: "How long to wait before force-closing a connection." },
           "host"               => { type: String,
+                                    external: true,
                                     desc: "Host name." },
           "host-key-check"     => { type: TrueClass,
+                                    external: true,
                                     desc: "Whether to perform host key validation when connecting." },
           "extensions"         => { type: Array,
                                     desc: "List of file extensions that are accepted for scripts or tasks on Windows. "\
@@ -26,6 +35,7 @@ module Bolt
                                          "a `.py` script runs with `python.exe`. The extensions `.ps1`, `.rb`, and "\
                                          "`.pp` are always allowed and run via hard-coded executables." },
           "interpreters"       => { type: Hash,
+                                    external: true,
                                     desc: "A map of an extension name to the absolute path of an executable, "\
                                           "enabling you to override the shebang defined in a task executable. The "\
                                           "extension can optionally be specified with the `.` character (`.py` and "\
@@ -41,26 +51,36 @@ module Bolt
           "password"           => { type: String,
                                     desc: "Login password." },
           "port"               => { type: Integer,
+                                    external: true,
                                     desc: "Connection port." },
-          "private-key"        => { desc: "Either the path to the private key file to use for authentication, or a "\
+          "private-key"        => { external: true,
+                                    desc: "Either the path to the private key file to use for authentication, or a "\
                                           "hash with the key `key-data` and the contents of the private key." },
           "proxyjump"          => { type: String,
                                     desc: "A jump host to proxy connections through, and an optional user to "\
                                           "connect with." },
           "run-as"             => { type: String,
+                                    external: true,
                                     desc: "A different user to run commands as after login." },
           "run-as-command"     => { type: Array,
+                                    external: true,
                                     desc: "The command to elevate permissions. Bolt appends the user and command "\
                                           "strings to the configured `run-as-command` before running it on the "\
                                           "target. This command must not require an interactive password prompt, "\
                                           "and the `sudo-password` option is ignored when `run-as-command` is "\
                                           "specified. The `run-as-command` must be specified as an array." },
           "script-dir"         => { type: String,
+                                    external: true,
                                     desc: "The subdirectory of the tmpdir to use in place of a randomized "\
                                           "subdirectory for uploading and executing temporary files on the "\
                                           "target. It's expected that this directory already exists as a subdir "\
                                           "of tmpdir, which is either configured or defaults to `/tmp`." },
+          "ssh-command"        => { external: true,
+                                    desc: "Command and flags to use when SSHing. This enables the external "\
+                                          "SSH transport which shells out to the specified command. "\
+                                          "**This option is experimental.**" },
           "sudo-executable"    => { type: String,
+                                    external: true,
                                     desc: "The executable to use when escalating to the configured `run-as` "\
                                           "user. This is useful when you want to escalate using the configured "\
                                           "`sudo-password`, since `run-as-command` does not use `sudo-password` "\
@@ -68,14 +88,17 @@ module Bolt
                                           "`<sudo-executable> -S -u <user> -p custom_bolt_prompt <command>`. "\
                                           "**This option is experimental.**" },
           "sudo-password"      => { type: String,
+                                    external: true,
                                     desc: "Password to use when changing users via `run-as`." },
           "tmpdir"             => { type: String,
+                                    external: true,
                                     desc: "The directory to upload and execute temporary files on the target." },
           "tty"                => { type: TrueClass,
                                     desc: "Request a pseudo tty for the session. This option is generally "\
                                           "only used in conjunction with the `run-as` option when the sudoers "\
                                           "policy requires a `tty`." },
           "user"               => { type: String,
+                                    external: true,
                                     desc: "Login user." }
         }.freeze
 
@@ -100,6 +123,13 @@ module Bolt
 
             if key_opt.instance_of?(String)
               @config['private-key'] = File.expand_path(key_opt, @project)
+
+              # We have an explicit test for this to only warn if using net-ssh transport
+              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['ssh-command']
+            end
+
+            if key_opt.instance_of?(Hash) && @config['ssh-command']
+              raise Bolt::ValidationError, 'private-key must be a filepath when using ssh-command'
             end
           end
 
@@ -127,6 +157,25 @@ module Bolt
               end
             end
           end
+
+          if @config['ssh-command'] && !@config['load-config']
+            msg = 'Cannot use external SSH transport with load-config set to false'
+            raise Bolt::ValidationError, msg
+          end
+
+          if (ssh_cmd = @config['ssh-command'])
+            unless ssh_cmd.is_a?(String) || ssh_cmd.is_a?(Array)
+              raise Bolt::ValidationError,
+                    "ssh-command must be a String or Array, received #{ssh_cmd.class} #{ssh_cmd.inspect}"
+            end
+          end
+
+          if (copy_cmd = @config['copy-command'])
+            unless copy_cmd.is_a?(String) || copy_cmd.is_a?(Array)
+              raise Bolt::ValidationError,
+                    "copy-command must be a String or Array, received #{copy_cmd.class} #{copy_cmd.inspect}"
+            end
+          end
         end
       end
     end