bolt 2.6.0 → 2.11.0

data/lib/bolt/config.rb

@@ -3,7 +3,7 @@
 require 'etc'
 require 'logging'
 require 'pathname'
-require 'bolt/boltdir'
+require 'bolt/project'
 require 'bolt/logger'
 require 'bolt/util'
 # Transport config objects
@@ -23,7 +23,7 @@ module Bolt
   end
 
   class Config
-    attr_reader :config_files, :warnings, :data, :transports, :boltdir
+    attr_reader :config_files, :warnings, :data, :transports, :project
 
     TRANSPORT_CONFIG = {
       'ssh'    => Bolt::Config::Transport::SSH,
@@ -34,6 +34,8 @@ module Bolt
       'remote' => Bolt::Config::Transport::Remote
     }.freeze
 
+    # NOTE: All configuration options should have a corresponding schema property
+    #       in schemas/bolt-config.schema.json
     OPTIONS = {
       "apply_settings"           => "A map of Puppet settings to use when applying Puppet code",
       "color"                    => "Whether to use colored output when printing messages to the console.",
@@ -64,8 +66,8 @@ module Bolt
 
     DEFAULT_OPTIONS = {
       "color" => true,
-      "concurrency" => 100,
       "compile-concurrency" => "Number of cores",
+      "concurrency" => "100 or one-third of the ulimit, whichever is lower",
       "format" => "human",
       "hiera-config" => "Boltdir/hiera.yaml",
       "inventoryfile" => "Boltdir/inventory.yaml",
@@ -101,31 +103,33 @@ module Bolt
       "show_diff" => false
     }.freeze
 
+    DEFAULT_DEFAULT_CONCURRENCY = 100
+
     def self.default
-      new(Bolt::Boltdir.new('.'), {})
+      new(Bolt::Project.new('.'), {})
     end
 
-    def self.from_boltdir(boltdir, overrides = {})
+    def self.from_project(project, overrides = {})
       data = {
-        filepath: boltdir.config_file,
-        data: Bolt::Util.read_optional_yaml_hash(boltdir.config_file, 'config')
+        filepath: project.config_file,
+        data: Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
       }
 
       data = load_defaults.push(data).select { |config| config[:data]&.any? }
 
-      new(boltdir, data, overrides)
+      new(project, data, overrides)
     end
 
     def self.from_file(configfile, overrides = {})
-      boltdir = Bolt::Boltdir.new(Pathname.new(configfile).expand_path.dirname)
+      project = Bolt::Project.new(Pathname.new(configfile).expand_path.dirname)
 
       data = {
-        filepath: boltdir.config_file,
+        filepath: project.config_file,
         data: Bolt::Util.read_yaml_hash(configfile, 'config')
       }
       data = load_defaults.push(data).select { |config| config[:data]&.any? }
 
-      new(boltdir, data, overrides)
+      new(project, data, overrides)
     end
 
     def self.load_defaults
@@ -148,14 +152,14 @@ module Bolt
       confs
     end
 
-    def initialize(boltdir, config_data, overrides = {})
+    def initialize(project, config_data, overrides = {})
       unless config_data.is_a?(Array)
-        config_data = [{ filepath: boltdir.config_file, data: config_data }]
+        config_data = [{ filepath: project.config_file, data: config_data }]
       end
 
       @logger = Logging.logger[self]
       @warnings = []
-      @boltdir = boltdir
+      @project = project
       @transports = {}
       @config_files = []
 
@@ -163,7 +167,7 @@ module Bolt
         'apply_settings'      => {},
         'color'               => true,
         'compile-concurrency' => Etc.nprocessors,
-        'concurrency'         => 100,
+        'concurrency'         => default_concurrency,
         'format'              => 'human',
         'log'                 => { 'console' => {} },
         'plugin_hooks'        => {},
@@ -181,10 +185,22 @@ module Bolt
 
       override_data = normalize_overrides(overrides)
 
+      # If we need to lower concurrency and concurrency is not configured
+      ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
+      if default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+         !ld_concurrency &&
+         !override_data.key?('concurrency')
+        concurrency_warning = { option: 'concurrency',
+                                msg: "Concurrency will default to #{default_concurrency} because ulimit "\
+                                "is low: #{Etc.sysconf(Etc::SC_OPEN_MAX)}. Set concurrency with "\
+                                "'--concurrency', or set your ulimit with 'ulimit -n <limit>'" }
+        @warnings << concurrency_warning
+      end
+
       @data = merge_config_layers(default_data, *loaded_data, override_data)
 
       TRANSPORT_CONFIG.each do |transport, config|
-        @transports[transport] = config.new(@data.delete(transport), @boltdir.path)
+        @transports[transport] = config.new(@data.delete(transport), @project.path)
       end
 
       finalize_data
@@ -250,7 +266,7 @@ module Bolt
         @data['log'] = update_logs(@data['log'])
       end
 
-      # Expand paths relative to the Boltdir. Any settings that came from the
+      # Expand paths relative to the project. Any settings that came from the
       # CLI will already be absolute, so the expand will be skipped.
       if @data.key?('modulepath')
         moduledirs = if data['modulepath'].is_a?(String)
@@ -259,12 +275,12 @@ module Bolt
                        data['modulepath']
                      end
         @data['modulepath'] = moduledirs.map do |moduledir|
-          File.expand_path(moduledir, @boltdir.path)
+          File.expand_path(moduledir, @project.path)
         end
       end
 
       %w[hiera-config inventoryfile trusted-external-command].each do |opt|
-        @data[opt] = File.expand_path(@data[opt], @boltdir.path) if @data.key?(opt)
+        @data[opt] = File.expand_path(@data[opt], @project.path) if @data.key?(opt)
       end
 
       # Filter hashes to only include valid options
@@ -275,7 +291,7 @@ module Bolt
     private def normalize_log(target)
       return target if target == 'console'
       target = target[5..-1] if target.start_with?('file:')
-      'file:' + File.expand_path(target, @boltdir.path)
+      'file:' + File.expand_path(target, @project.path)
     end
 
     private def update_logs(logs)
@@ -310,10 +326,10 @@ module Bolt
         @warnings << { option: 'future', msg: msg }
       end
 
-      keys = OPTIONS.keys - %w[plugins plugin_hooks]
+      keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
-        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks]
+        valid_keys = TRANSPORT_CONFIG.keys + %w[plugins plugin_hooks puppetdb]
         raise Bolt::ValidationError,
               "Found unsupported key _plugin in config setting #{key}. Plugins are only available in "\
               "#{valid_keys.join(', ')}."
@@ -348,23 +364,23 @@ module Bolt
     end
 
     def default_inventoryfile
-      @boltdir.inventory_file
+      @project.inventory_file
     end
 
     def rerunfile
-      @boltdir.rerunfile
+      @project.rerunfile
     end
 
     def hiera_config
-      @data['hiera-config'] || @boltdir.hiera_config
+      @data['hiera-config'] || @project.hiera_config
     end
 
     def puppetfile
-      @puppetfile || @boltdir.puppetfile
+      @puppetfile || @project.puppetfile
     end
 
     def modulepath
-      @data['modulepath'] || @boltdir.modulepath
+      @data['modulepath'] || @project.modulepath
     end
 
     def modulepath=(value)
@@ -456,5 +472,13 @@ module Bolt
         l =~ /[A-Za-z]/ ? "[#{l.upcase}#{l.downcase}]" : l
       end.join
     end
+
+    def default_concurrency
+      if Bolt::Util.windows? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300 || Etc.sysconf(Etc::SC_OPEN_MAX).nil?
+        DEFAULT_DEFAULT_CONCURRENCY
+      else
+        (Etc.sysconf(Etc::SC_OPEN_MAX) / 3).floor
+      end
+    end
   end
 end

data/lib/bolt/config/transport/base.rb

@@ -9,12 +9,12 @@ module Bolt
       class Base
         attr_reader :input
 
-        def initialize(data = {}, boltdir = nil)
+        def initialize(data = {}, project = nil)
           assert_hash_or_config(data)
           @input    = data
           @resolved = !Bolt::Util.references?(input)
           @config   = resolved? ? Bolt::Util.deep_merge(defaults, filter(input)) : defaults
-          @boltdir  = boltdir
+          @project  = project
 
           validate if resolved?
         end
@@ -62,7 +62,7 @@ module Bolt
             Bolt::Util.deep_merge(acc, layer_data)
           end
 
-          self.class.new(merged, @boltdir)
+          self.class.new(merged, @project)
         end
 
         # Resolve any references in the input data, then remerge it with the defaults

data/lib/bolt/config/transport/docker.rb

@@ -7,7 +7,11 @@ module Bolt
   class Config
     module Transport
       class Docker < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
+          "cleanup"       => { type: TrueClass,
+                               desc: "Whether to clean up temporary files created on targets." },
           "host"          => { type: String,
                                desc: "Host name." },
           "interpreters"  => { type: Hash,
@@ -27,7 +31,9 @@ module Bolt
                                desc: "Whether to enable tty on exec commands." }
         }.freeze
 
-        DEFAULTS = {}.freeze
+        DEFAULTS = {
+          'cleanup' => true
+        }.freeze
 
         private def validate
           super

data/lib/bolt/config/transport/local.rb

@@ -7,7 +7,11 @@ module Bolt
   class Config
     module Transport
       class Local < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
+          "cleanup"         => { type: TrueClass,
+                                 desc: "Whether to clean up temporary files created on targets." },
           "interpreters"    => { type: Hash,
                                  desc: "A map of an extension name to the absolute path of an executable, "\
                                       "enabling you to override the shebang defined in a task executable. The "\
@@ -36,6 +40,8 @@ module Bolt
         }.freeze
 
         WINDOWS_OPTIONS = {
+          "cleanup"      => { type: TrueClass,
+                              desc: "Whether to clean up temporary files created on targets." },
           "interpreters" => { type: Hash,
                               desc: "A map of an extension name to the absolute path of an executable, "\
                                     "enabling you to override the shebang defined in a task executable. The "\
@@ -47,7 +53,9 @@ module Bolt
                               desc: "The directory to copy and execute temporary files." }
         }.freeze
 
-        DEFAULTS = {}.freeze
+        DEFAULTS = {
+          'cleanup' => true
+        }.freeze
 
         def self.options
           Bolt::Util.windows? ? WINDOWS_OPTIONS : OPTIONS

data/lib/bolt/config/transport/orch.rb

@@ -7,6 +7,8 @@ module Bolt
   class Config
     module Transport
       class Orch < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "cacert"            => { type: String,
                                    desc: "The path to the CA certificate." },
@@ -32,12 +34,12 @@ module Bolt
           super
 
           if @config['cacert']
-            @config['cacert'] = File.expand_path(@config['cacert'], @boltdir)
+            @config['cacert'] = File.expand_path(@config['cacert'], @project)
             Bolt::Util.validate_file('cacert', @config['cacert'])
           end
 
           if @config['token-file']
-            @config['token-file'] = File.expand_path(@config['token-file'], @boltdir)
+            @config['token-file'] = File.expand_path(@config['token-file'], @project)
             Bolt::Util.validate_file('token-file', @config['token-file'])
           end
         end

data/lib/bolt/config/transport/remote.rb

@@ -7,6 +7,8 @@ module Bolt
   class Config
     module Transport
       class Remote < Base
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
           "run-on" => { type: String,
                         desc: "The proxy target that the task executes on." }

data/lib/bolt/config/transport/ssh.rb

@@ -7,16 +7,35 @@ module Bolt
   class Config
     module Transport
       class SSH < Base
+        LOGIN_SHELLS = %w[sh bash zsh dash ksh powershell].freeze
+
+        # NOTE: All transport configuration options should have a corresponding schema definition
+        #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
+          "cleanup"            => { type: TrueClass,
+                                    external: true,
+                                    desc: "Whether to clean up temporary files created on targets." },
           "connect-timeout"    => { type: Integer,
                                     desc: "How long to wait when establishing connections." },
+          "copy-command"       => { external: true,
+                                    desc: "Command to use when copying files using ssh-command. "\
+                                          "Bolt runs `<copy-command> <src> <dest>`. **This option is experimental.**" },
           "disconnect-timeout" => { type: Integer,
                                     desc: "How long to wait before force-closing a connection." },
           "host"               => { type: String,
+                                    external: true,
                                     desc: "Host name." },
           "host-key-check"     => { type: TrueClass,
+                                    external: true,
                                     desc: "Whether to perform host key validation when connecting." },
+          "extensions"         => { type: Array,
+                                    desc: "List of file extensions that are accepted for scripts or tasks on Windows. "\
+                                         "Scripts with these file extensions rely on the target's file type "\
+                                         "association to run. For example, if Python is installed on the system, "\
+                                         "a `.py` script runs with `python.exe`. The extensions `.ps1`, `.rb`, and "\
+                                         "`.pp` are always allowed and run via hard-coded executables." },
           "interpreters"       => { type: Hash,
+                                    external: true,
                                     desc: "A map of an extension name to the absolute path of an executable, "\
                                           "enabling you to override the shebang defined in a task executable. The "\
                                           "extension can optionally be specified with the `.` character (`.py` and "\
@@ -25,29 +44,43 @@ module Bolt
                                           "Bolt Ruby interpreter by default." },
           "load-config"        => { type: TrueClass,
                                     desc: "Whether to load system SSH configuration." },
+          "login-shell"        => { type: String,
+                                    desc: "Which login shell Bolt should expect on the target. "\
+                                          "Supported shells are #{LOGIN_SHELLS.join(', ')}. "\
+                                          "**This option is experimental.**" },
           "password"           => { type: String,
                                     desc: "Login password." },
           "port"               => { type: Integer,
+                                    external: true,
                                     desc: "Connection port." },
-          "private-key"        => { desc: "Either the path to the private key file to use for authentication, or a "\
+          "private-key"        => { external: true,
+                                    desc: "Either the path to the private key file to use for authentication, or a "\
                                           "hash with the key `key-data` and the contents of the private key." },
           "proxyjump"          => { type: String,
                                     desc: "A jump host to proxy connections through, and an optional user to "\
                                           "connect with." },
           "run-as"             => { type: String,
+                                    external: true,
                                     desc: "A different user to run commands as after login." },
           "run-as-command"     => { type: Array,
+                                    external: true,
                                     desc: "The command to elevate permissions. Bolt appends the user and command "\
                                           "strings to the configured `run-as-command` before running it on the "\
                                           "target. This command must not require an interactive password prompt, "\
                                           "and the `sudo-password` option is ignored when `run-as-command` is "\
                                           "specified. The `run-as-command` must be specified as an array." },
           "script-dir"         => { type: String,
+                                    external: true,
                                     desc: "The subdirectory of the tmpdir to use in place of a randomized "\
                                           "subdirectory for uploading and executing temporary files on the "\
                                           "target. It's expected that this directory already exists as a subdir "\
                                           "of tmpdir, which is either configured or defaults to `/tmp`." },
+          "ssh-command"        => { external: true,
+                                    desc: "Command and flags to use when SSHing. This enables the external "\
+                                          "SSH transport which shells out to the specified command. "\
+                                          "**This option is experimental.**" },
           "sudo-executable"    => { type: String,
+                                    external: true,
                                     desc: "The executable to use when escalating to the configured `run-as` "\
                                           "user. This is useful when you want to escalate using the configured "\
                                           "`sudo-password`, since `run-as-command` does not use `sudo-password` "\
@@ -55,22 +88,27 @@ module Bolt
                                           "`<sudo-executable> -S -u <user> -p custom_bolt_prompt <command>`. "\
                                           "**This option is experimental.**" },
           "sudo-password"      => { type: String,
+                                    external: true,
                                     desc: "Password to use when changing users via `run-as`." },
           "tmpdir"             => { type: String,
+                                    external: true,
                                     desc: "The directory to upload and execute temporary files on the target." },
           "tty"                => { type: TrueClass,
                                     desc: "Request a pseudo tty for the session. This option is generally "\
                                           "only used in conjunction with the `run-as` option when the sudoers "\
                                           "policy requires a `tty`." },
           "user"               => { type: String,
+                                    external: true,
                                     desc: "Login user." }
         }.freeze
 
         DEFAULTS = {
+          "cleanup"            => true,
           "connect-timeout"    => 10,
           "tty"                => false,
           "load-config"        => true,
-          "disconnect-timeout" => 5
+          "disconnect-timeout" => 5,
+          "login-shell"        => 'bash'
         }.freeze
 
         private def validate
@@ -84,7 +122,14 @@ module Bolt
             end
 
             if key_opt.instance_of?(String)
-              @config['private-key'] = File.expand_path(key_opt, @boltdir)
+              @config['private-key'] = File.expand_path(key_opt, @project)
+
+              # We have an explicit test for this to only warn if using net-ssh transport
+              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['ssh-command']
+            end
+
+            if key_opt.instance_of?(Hash) && @config['ssh-command']
+              raise Bolt::ValidationError, 'private-key must be a filepath when using ssh-command'
             end
           end
 
@@ -92,12 +137,45 @@ module Bolt
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
 
+          if @config['login-shell'] && !LOGIN_SHELLS.include?(@config['login-shell'])
+            raise Bolt::ValidationError,
+                  "Unsupported login-shell #{@config['login-shell']}. Supported shells are #{LOGIN_SHELLS.join(', ')}"
+          end
+
           if (run_as_cmd = @config['run-as-command'])
             unless run_as_cmd.all? { |n| n.is_a?(String) }
               raise Bolt::ValidationError,
                     "run-as-command must be an Array of Strings, received #{run_as_cmd.class} #{run_as_cmd.inspect}"
             end
           end
+
+          if @config['login-shell'] == 'powershell'
+            %w[tty run-as].each do |key|
+              if @config[key]
+                raise Bolt::ValidationError,
+                      "#{key} is not supported when using PowerShell"
+              end
+            end
+          end
+
+          if @config['ssh-command'] && !@config['load-config']
+            msg = 'Cannot use external SSH transport with load-config set to false'
+            raise Bolt::ValidationError, msg
+          end
+
+          if (ssh_cmd = @config['ssh-command'])
+            unless ssh_cmd.is_a?(String) || ssh_cmd.is_a?(Array)
+              raise Bolt::ValidationError,
+                    "ssh-command must be a String or Array, received #{ssh_cmd.class} #{ssh_cmd.inspect}"
+            end
+          end
+
+          if (copy_cmd = @config['copy-command'])
+            unless copy_cmd.is_a?(String) || copy_cmd.is_a?(Array)
+              raise Bolt::ValidationError,
+                    "copy-command must be a String or Array, received #{copy_cmd.class} #{copy_cmd.inspect}"
+            end
+          end
         end
       end
     end