bolt 2.13.0 → 2.18.0

data/lib/bolt/config/transport/orch.rb

@@ -7,24 +7,15 @@ module Bolt
   class Config
     module Transport
       class Orch < Base
-        # NOTE: All transport configuration options should have a corresponding schema definition
-        #       in schemas/bolt-transport-definitions.json
-        OPTIONS = {
-          "cacert"            => { type: String,
-                                   desc: "The path to the CA certificate." },
-          "host"              => { type: String,
-                                   desc: "Host name." },
-          "job-poll-interval" => { type: Integer,
-                                   desc: "Set interval to poll orchestrator for job status." },
-          "job-poll-timeout"  => { type: Integer,
-                                   desc: "Set time to wait for orchestrator job status." },
-          "service-url"       => { type: String,
-                                   desc: "The URL of the orchestrator API." },
-          "task-environment"  => { type: String,
-                                   desc: "The environment the orchestrator loads task code from." },
-          "token-file"        => { type: String,
-                                   desc: "The path to the token file." }
-        }.freeze
+        OPTIONS = %w[
+          cacert
+          host
+          job-poll-interval
+          job-poll-timeout
+          service-url
+          task-environment
+          token-file
+        ].freeze
 
         DEFAULTS = {
           "task-environment" => "production"

data/lib/bolt/config/transport/remote.rb

@@ -7,12 +7,9 @@ module Bolt
   class Config
     module Transport
       class Remote < Base
-        # NOTE: All transport configuration options should have a corresponding schema definition
-        #       in schemas/bolt-transport-definitions.json
-        OPTIONS = {
-          "run-on" => { type: String,
-                        desc: "The proxy target that the task executes on." }
-        }.freeze
+        OPTIONS = %w[
+          run-on
+        ].freeze
 
         DEFAULTS = {
           "run-on" => "localhost"

data/lib/bolt/config/transport/ssh.rb

@@ -7,110 +7,82 @@ module Bolt
   class Config
     module Transport
       class SSH < Base
-        LOGIN_SHELLS = %w[sh bash zsh dash ksh powershell].freeze
-
-        # NOTE: All transport configuration options should have a corresponding schema definition
-        #       in schemas/bolt-transport-definitions.json
-        OPTIONS = {
-          "cleanup"            => { type: TrueClass,
-                                    external: true,
-                                    desc: "Whether to clean up temporary files created on targets." },
-          "connect-timeout"    => { type: Integer,
-                                    desc: "How long to wait when establishing connections." },
-          "copy-command"       => { external: true,
-                                    desc: "Command to use when copying files using ssh-command. "\
-                                          "Bolt runs `<copy-command> <src> <dest>`. **This option is experimental.**" },
-          "disconnect-timeout" => { type: Integer,
-                                    desc: "How long to wait before force-closing a connection." },
-          "host"               => { type: String,
-                                    external: true,
-                                    desc: "Host name." },
-          "host-key-check"     => { type: TrueClass,
-                                    external: true,
-                                    desc: "Whether to perform host key validation when connecting." },
-          "extensions"         => { type: Array,
-                                    desc: "List of file extensions that are accepted for scripts or tasks on Windows. "\
-                                         "Scripts with these file extensions rely on the target's file type "\
-                                         "association to run. For example, if Python is installed on the system, "\
-                                         "a `.py` script runs with `python.exe`. The extensions `.ps1`, `.rb`, and "\
-                                         "`.pp` are always allowed and run via hard-coded executables." },
-          "interpreters"       => { type: Hash,
-                                    external: true,
-                                    desc: "A map of an extension name to the absolute path of an executable, "\
-                                          "enabling you to override the shebang defined in a task executable. The "\
-                                          "extension can optionally be specified with the `.` character (`.py` and "\
-                                          "`py` both map to a task executable `task.py`) and the extension is case "\
-                                          "sensitive. When a target's name is `localhost`, Ruby tasks run with the "\
-                                          "Bolt Ruby interpreter by default." },
-          "load-config"        => { type: TrueClass,
-                                    desc: "Whether to load system SSH configuration." },
-          "login-shell"        => { type: String,
-                                    desc: "Which login shell Bolt should expect on the target. "\
-                                          "Supported shells are #{LOGIN_SHELLS.join(', ')}. "\
-                                          "**This option is experimental.**" },
-          "password"           => { type: String,
-                                    desc: "Login password." },
-          "port"               => { type: Integer,
-                                    external: true,
-                                    desc: "Connection port." },
-          "private-key"        => { external: true,
-                                    desc: "Either the path to the private key file to use for authentication, or a "\
-                                          "hash with the key `key-data` and the contents of the private key." },
-          "proxyjump"          => { type: String,
-                                    desc: "A jump host to proxy connections through, and an optional user to "\
-                                          "connect with." },
-          "run-as"             => { type: String,
-                                    external: true,
-                                    desc: "A different user to run commands as after login." },
-          "run-as-command"     => { type: Array,
-                                    external: true,
-                                    desc: "The command to elevate permissions. Bolt appends the user and command "\
-                                          "strings to the configured `run-as-command` before running it on the "\
-                                          "target. This command must not require an interactive password prompt, "\
-                                          "and the `sudo-password` option is ignored when `run-as-command` is "\
-                                          "specified. The `run-as-command` must be specified as an array." },
-          "script-dir"         => { type: String,
-                                    external: true,
-                                    desc: "The subdirectory of the tmpdir to use in place of a randomized "\
-                                          "subdirectory for uploading and executing temporary files on the "\
-                                          "target. It's expected that this directory already exists as a subdir "\
-                                          "of tmpdir, which is either configured or defaults to `/tmp`." },
-          "ssh-command"        => { external: true,
-                                    desc: "Command and flags to use when SSHing. This enables the external "\
-                                          "SSH transport which shells out to the specified command. "\
-                                          "**This option is experimental.**" },
-          "sudo-executable"    => { type: String,
-                                    external: true,
-                                    desc: "The executable to use when escalating to the configured `run-as` "\
-                                          "user. This is useful when you want to escalate using the configured "\
-                                          "`sudo-password`, since `run-as-command` does not use `sudo-password` "\
-                                          "or support prompting. The command executed on the target is "\
-                                          "`<sudo-executable> -S -u <user> -p custom_bolt_prompt <command>`. "\
-                                          "**This option is experimental.**" },
-          "sudo-password"      => { type: String,
-                                    external: true,
-                                    desc: "Password to use when changing users via `run-as`." },
-          "tmpdir"             => { type: String,
-                                    external: true,
-                                    desc: "The directory to upload and execute temporary files on the target." },
-          "tty"                => { type: TrueClass,
-                                    desc: "Request a pseudo tty for the session. This option is generally "\
-                                          "only used in conjunction with the `run-as` option when the sudoers "\
-                                          "policy requires a `tty`." },
-          "user"               => { type: String,
-                                    external: true,
-                                    desc: "Login user." }
-        }.freeze
+        # Options available when using the net-ssh-based transport
+        OPTIONS = %w[
+          cleanup
+          connect-timeout
+          disconnect-timeout
+          encryption-algorithms
+          extensions
+          host
+          host-key-algorithms
+          host-key-check
+          interpreters
+          kex-algorithms
+          load-config
+          login-shell
+          mac-algorithms
+          password
+          port
+          private-key
+          proxyjump
+          script-dir
+          tmpdir
+          tty
+          user
+        ].concat(RUN_AS_OPTIONS).sort.freeze
+
+        # Options available when using the native ssh transport
+        NATIVE_OPTIONS = %w[
+          cleanup
+          copy-command
+          host
+          host-key-check
+          interpreters
+          native-ssh
+          port
+          private-key
+          script-dir
+          ssh-command
+          tmpdir
+          user
+        ].concat(RUN_AS_OPTIONS).sort.freeze
 
         DEFAULTS = {
           "cleanup"            => true,
           "connect-timeout"    => 10,
-          "tty"                => false,
-          "load-config"        => true,
           "disconnect-timeout" => 5,
-          "login-shell"        => 'bash'
+          "load-config"        => true,
+          "login-shell"        => 'bash',
+          "tty"                => false
         }.freeze
 
+        # The set of options available for the ssh and native ssh transports overlap, so we
+        # need to check which transport is used before fully initializing, otherwise options
+        # may not be filtered correctly.
+        def initialize(data = {}, project = nil)
+          assert_hash_or_config(data)
+          @native = true if data['native-ssh']
+          super(data, project)
+        end
+
+        # This method is used to filter CLI options in the Config class. This
+        # should include `ssh-command` so that we can later warn if the option
+        # is present without `native-ssh`
+        def self.options
+          %w[ssh-command native-ssh].concat(OPTIONS)
+        end
+
+        private def filter(unfiltered)
+          # Because we filter before merging config together it's impossible to
+          # know whether both ssh-command *and* native-ssh will be specified
+          # unless they are both in the filter. However, we can't add
+          # ssh-command to OPTIONS since that's used for documenting available
+          # options. This makes it so that ssh-command is preserved so we can
+          # warn once all config is resolved if native-ssh isn't set.
+          @native ? unfiltered.slice(*NATIVE_OPTIONS) : unfiltered.slice(*self.class.options)
+        end
+
         private def validate
           super
 
@@ -125,11 +97,11 @@ module Bolt
               @config['private-key'] = File.expand_path(key_opt, @project)
 
               # We have an explicit test for this to only warn if using net-ssh transport
-              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['ssh-command']
+              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['native-ssh']
             end
 
-            if key_opt.instance_of?(Hash) && @config['ssh-command']
-              raise Bolt::ValidationError, 'private-key must be a filepath when using ssh-command'
+            if key_opt.instance_of?(Hash) && @config['native-ssh']
+              raise Bolt::ValidationError, 'private-key must be a filepath when using native-ssh'
             end
           end
 
@@ -142,10 +114,11 @@ module Bolt
                   "Unsupported login-shell #{@config['login-shell']}. Supported shells are #{LOGIN_SHELLS.join(', ')}"
           end
 
-          if (run_as_cmd = @config['run-as-command'])
-            unless run_as_cmd.all? { |n| n.is_a?(String) }
+          %w[encryption-algorithms host-key-algorithms kex-algorithms mac-algorithms run-as-command].each do |opt|
+            next unless @config.key?(opt)
+            unless @config[opt].all? { |n| n.is_a?(String) }
               raise Bolt::ValidationError,
-                    "run-as-command must be an Array of Strings, received #{run_as_cmd.class} #{run_as_cmd.inspect}"
+                    "#{opt} must be an Array of Strings, received #{@config[opt].inspect}"
             end
           end
 
@@ -158,24 +131,10 @@ module Bolt
             end
           end
 
-          if @config['ssh-command'] && !@config['load-config']
-            msg = 'Cannot use external SSH transport with load-config set to false'
+          if @config['native-ssh'] && !@config['load-config']
+            msg = 'Cannot use native SSH transport with load-config set to false'
             raise Bolt::ValidationError, msg
           end
-
-          if (ssh_cmd = @config['ssh-command'])
-            unless ssh_cmd.is_a?(String) || ssh_cmd.is_a?(Array)
-              raise Bolt::ValidationError,
-                    "ssh-command must be a String or Array, received #{ssh_cmd.class} #{ssh_cmd.inspect}"
-            end
-          end
-
-          if (copy_cmd = @config['copy-command'])
-            unless copy_cmd.is_a?(String) || copy_cmd.is_a?(Array)
-              raise Bolt::ValidationError,
-                    "copy-command must be a String or Array, received #{copy_cmd.class} #{copy_cmd.inspect}"
-            end
-          end
         end
       end
     end

data/lib/bolt/config/transport/winrm.rb

@@ -7,53 +7,24 @@ module Bolt
   class Config
     module Transport
       class WinRM < Base
-        # NOTE: All transport configuration options should have a corresponding schema definition
-        #       in schemas/bolt-transport-definitions.json
-        OPTIONS = {
-          "basic-auth-only" => { type: TrueClass,
-                                 desc: "Force basic authentication. This option is only available when using SSL." },
-          "cacert"          => { type: String,
-                                 desc: "The path to the CA certificate." },
-          "cleanup"         => { type: TrueClass,
-                                 desc: "Whether to clean up temporary files created on targets." },
-          "connect-timeout" => { type: Integer,
-                                 desc: "How long Bolt should wait when establishing connections." },
-          "extensions"      => { type: Array,
-                                 desc: "List of file extensions that are accepted for scripts or tasks. "\
-                                      "Scripts with these file extensions rely on the target's file type "\
-                                      "association to run. For example, if Python is installed on the system, "\
-                                      "a `.py` script runs with `python.exe`. The extensions `.ps1`, `.rb`, and "\
-                                      "`.pp` are always allowed and run via hard-coded executables." },
-          "file-protocol"   => { type: String,
-                                 desc: "Which file transfer protocol to use. Either `winrm` or `smb`. Using `smb` is "\
-                                      "recommended for large file transfers." },
-          "host"            => { type: String,
-                                 desc: "Host name." },
-          "interpreters"    => { type: Hash,
-                                 desc: "A map of an extension name to the absolute path of an executable, "\
-                                      "enabling you to override the shebang defined in a task executable. The "\
-                                      "extension can optionally be specified with the `.` character (`.py` and "\
-                                      "`py` both map to a task executable `task.py`) and the extension is case "\
-                                      "sensitive. When a target's name is `localhost`, Ruby tasks run with the "\
-                                      "Bolt Ruby interpreter by default." },
-          "password"        => { type: String,
-                                 desc: "Login password. **Required unless using Kerberos.**" },
-          "port"            => { type: Integer,
-                                 desc: "Connection port." },
-          "realm"           => { type: String,
-                                 desc: "Kerberos realm (Active Directory domain) to authenticate against." },
-          "smb-port"        => { type: Integer,
-                                 desc: "With file-protocol set to smb, this is the port to establish a "\
-                                      "connection on." },
-          "ssl"             => { type: TrueClass,
-                                 desc: "When true, Bolt uses secure https connections for WinRM." },
-          "ssl-verify"      => { type: TrueClass,
-                                 desc: "When true, verifies the targets certificate matches the cacert." },
-          "tmpdir"          => { type: String,
-                                 desc: "The directory to upload and execute temporary files on the target." },
-          "user"            => { type: String,
-                                 desc: "Login user. **Required unless using Kerberos.**" }
-        }.freeze
+        OPTIONS = %w[
+          basic-auth-only
+          cacert
+          cleanup
+          connect-timeout
+          extensions
+          file-protocol
+          host
+          interpreters
+          password
+          port
+          realm
+          smb-port
+          ssl
+          ssl-verify
+          tmpdir
+          user
+        ].freeze
 
         DEFAULTS = {
           "basic-auth-only" => false,

data/lib/bolt/executor.rb

@@ -34,7 +34,8 @@ module Bolt
 
     def initialize(concurrency = 1,
                    analytics = Bolt::Analytics::NoopClient.new,
-                   noop = false)
+                   noop = false,
+                   modified_concurrency = false)
       # lazy-load expensive gem code
       require 'concurrent'
 
@@ -64,6 +65,9 @@ module Bolt
                 Concurrent.global_immediate_executor
               end
       @logger.debug { "Started with #{concurrency} max thread(s)" }
+
+      @concurrency = concurrency
+      @warn_concurrency = modified_concurrency
     end
 
     def transport(transport)
@@ -102,6 +106,15 @@ module Bolt
     # defined by the transport. Yields each batch, along with the corresponding
     # transport, to the block in turn and returns an array of result promises.
     def queue_execute(targets)
+      if @warn_concurrency && targets.length > @concurrency
+        @warn_concurrency = false
+        @logger.warn("The ulimit is low, which may cause file limit issues. Default concurrency has been set to "\
+                     "'#{@concurrency}' to mitigate those issues, which may cause Bolt to run slow. "\
+                     "Disable this warning by configuring ulimit using 'ulimit -n <limit>' in your shell "\
+                     "configuration, or by configuring Bolt's concurrency. "\
+                     "See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details.")
+      end
+
       targets.group_by(&:transport).flat_map do |protocol, protocol_targets|
         transport = transport(protocol)
         report_transport(transport, protocol_targets.count)

data/lib/bolt/inventory/group.rb

@@ -16,7 +16,7 @@ module Bolt
       DATA_KEYS = %w[config facts vars features plugin_hooks].freeze
       TARGET_KEYS = DATA_KEYS + %w[name alias uri]
       GROUP_KEYS = DATA_KEYS + %w[name groups targets]
-      CONFIG_KEYS = Bolt::Config::TRANSPORT_CONFIG.keys + ['transport']
+      CONFIG_KEYS = Bolt::Config::INVENTORY_OPTIONS.keys
 
       def initialize(input, plugins)
         @logger = Logging.logger[self]

data/lib/bolt/inventory/inventory.rb

@@ -78,20 +78,6 @@ module Bolt
         target_array.first
       end
 
-      def self.localhost_defaults(data)
-        defaults = {
-          'config' => {
-            'transport' => 'local',
-            'local' => { 'interpreters' => { '.rb' => RbConfig.ruby } }
-          },
-          'features' => ['puppet-agent']
-        }
-        data = Bolt::Util.deep_merge(defaults, data)
-        # If features is an empty array deep_merge won't add the puppet-agent
-        data['features'] += ['puppet-agent'] if data['features'].empty?
-        data
-      end
-
       #### PRIVATE ####
       def group_data_for(target_name)
         @groups.group_collect(target_name)
@@ -323,6 +309,10 @@ module Bolt
       def resources(target)
         @targets[target.name].resources
       end
+
+      def resource(target, type, title)
+        @targets[target.name].resource(type, title)
+      end
     end
   end
 end

data/lib/bolt/inventory/target.rb

@@ -30,6 +30,10 @@ module Bolt
           @safe_name = @uri_obj.omit(:password).to_str.sub(%r{^//}, '')
         end
 
+        if @name == 'localhost'
+          target_data = localhost_defaults(target_data)
+        end
+
         @config = target_data['config'] || {}
         @vars = target_data['vars'] || {}
         @facts = target_data['facts'] || {}
@@ -45,6 +49,20 @@ module Bolt
         validate
       end
 
+      def localhost_defaults(data)
+        defaults = {
+          'config' => {
+            'transport' => 'local',
+            'local' => { 'interpreters' => { '.rb' => RbConfig.ruby } }
+          },
+          'features' => ['puppet-agent']
+        }
+        data = Bolt::Util.deep_merge(defaults, data)
+        # If features is an empty array deep_merge won't add the puppet-agent
+        data['features'] += ['puppet-agent'] if data['features'].empty?
+        data
+      end
+
       # rubocop:disable Naming/AccessorMethodName
       def set_resource(resource)
         if (existing_resource = resources[resource.reference])
@@ -90,6 +108,10 @@ module Bolt
         end
       end
 
+      def resource(type, title)
+        resources[Bolt::ResourceInstance.format_reference(type, title)]
+      end
+
       def plugin_hooks
         # Merge plugin_hooks from the config file with any defined by the group
         # or assigned dynamically to the target
@@ -214,11 +236,6 @@ module Bolt
             'target_alias' => []
           }
 
-          # This should be handled by `get_targets`
-          if @name == 'localhost'
-            group_data = Bolt::Inventory::Inventory.localhost_defaults(group_data)
-          end
-
           @group_cache = group_data
         end