bolt 2.11.0 → 2.15.0

data/lib/bolt/config/transport/ssh.rb

@@ -12,94 +12,132 @@ module Bolt
         # NOTE: All transport configuration options should have a corresponding schema definition
         #       in schemas/bolt-transport-definitions.json
         OPTIONS = {
-          "cleanup"            => { type: TrueClass,
-                                    external: true,
-                                    desc: "Whether to clean up temporary files created on targets." },
-          "connect-timeout"    => { type: Integer,
-                                    desc: "How long to wait when establishing connections." },
-          "copy-command"       => { external: true,
-                                    desc: "Command to use when copying files using ssh-command. "\
-                                          "Bolt runs `<copy-command> <src> <dest>`. **This option is experimental.**" },
-          "disconnect-timeout" => { type: Integer,
-                                    desc: "How long to wait before force-closing a connection." },
-          "host"               => { type: String,
-                                    external: true,
-                                    desc: "Host name." },
-          "host-key-check"     => { type: TrueClass,
-                                    external: true,
-                                    desc: "Whether to perform host key validation when connecting." },
-          "extensions"         => { type: Array,
-                                    desc: "List of file extensions that are accepted for scripts or tasks on Windows. "\
-                                         "Scripts with these file extensions rely on the target's file type "\
-                                         "association to run. For example, if Python is installed on the system, "\
-                                         "a `.py` script runs with `python.exe`. The extensions `.ps1`, `.rb`, and "\
-                                         "`.pp` are always allowed and run via hard-coded executables." },
-          "interpreters"       => { type: Hash,
-                                    external: true,
-                                    desc: "A map of an extension name to the absolute path of an executable, "\
-                                          "enabling you to override the shebang defined in a task executable. The "\
-                                          "extension can optionally be specified with the `.` character (`.py` and "\
-                                          "`py` both map to a task executable `task.py`) and the extension is case "\
-                                          "sensitive. When a target's name is `localhost`, Ruby tasks run with the "\
-                                          "Bolt Ruby interpreter by default." },
-          "load-config"        => { type: TrueClass,
-                                    desc: "Whether to load system SSH configuration." },
-          "login-shell"        => { type: String,
-                                    desc: "Which login shell Bolt should expect on the target. "\
-                                          "Supported shells are #{LOGIN_SHELLS.join(', ')}. "\
-                                          "**This option is experimental.**" },
-          "password"           => { type: String,
-                                    desc: "Login password." },
-          "port"               => { type: Integer,
-                                    external: true,
-                                    desc: "Connection port." },
-          "private-key"        => { external: true,
-                                    desc: "Either the path to the private key file to use for authentication, or a "\
-                                          "hash with the key `key-data` and the contents of the private key." },
-          "proxyjump"          => { type: String,
-                                    desc: "A jump host to proxy connections through, and an optional user to "\
-                                          "connect with." },
-          "run-as"             => { type: String,
-                                    external: true,
-                                    desc: "A different user to run commands as after login." },
-          "run-as-command"     => { type: Array,
-                                    external: true,
-                                    desc: "The command to elevate permissions. Bolt appends the user and command "\
-                                          "strings to the configured `run-as-command` before running it on the "\
-                                          "target. This command must not require an interactive password prompt, "\
-                                          "and the `sudo-password` option is ignored when `run-as-command` is "\
-                                          "specified. The `run-as-command` must be specified as an array." },
-          "script-dir"         => { type: String,
-                                    external: true,
-                                    desc: "The subdirectory of the tmpdir to use in place of a randomized "\
-                                          "subdirectory for uploading and executing temporary files on the "\
-                                          "target. It's expected that this directory already exists as a subdir "\
-                                          "of tmpdir, which is either configured or defaults to `/tmp`." },
-          "ssh-command"        => { external: true,
-                                    desc: "Command and flags to use when SSHing. This enables the external "\
-                                          "SSH transport which shells out to the specified command. "\
-                                          "**This option is experimental.**" },
-          "sudo-executable"    => { type: String,
-                                    external: true,
-                                    desc: "The executable to use when escalating to the configured `run-as` "\
-                                          "user. This is useful when you want to escalate using the configured "\
-                                          "`sudo-password`, since `run-as-command` does not use `sudo-password` "\
-                                          "or support prompting. The command executed on the target is "\
-                                          "`<sudo-executable> -S -u <user> -p custom_bolt_prompt <command>`. "\
-                                          "**This option is experimental.**" },
-          "sudo-password"      => { type: String,
-                                    external: true,
-                                    desc: "Password to use when changing users via `run-as`." },
-          "tmpdir"             => { type: String,
-                                    external: true,
-                                    desc: "The directory to upload and execute temporary files on the target." },
-          "tty"                => { type: TrueClass,
-                                    desc: "Request a pseudo tty for the session. This option is generally "\
-                                          "only used in conjunction with the `run-as` option when the sudoers "\
-                                          "policy requires a `tty`." },
-          "user"               => { type: String,
-                                    external: true,
-                                    desc: "Login user." }
+          "cleanup"               => { type: TrueClass,
+                                       external: true,
+                                       desc: "Whether to clean up temporary files created on targets." },
+          "connect-timeout"       => { type: Integer,
+                                       desc: "How long to wait when establishing connections." },
+          "copy-command"          => { external: true,
+                                       desc: "Command to use when copying files using ssh-command. "\
+                                             "Bolt runs `<copy-command> <src> <dest>`. **This option is "\
+                                             "experimental.**" },
+          "disconnect-timeout"    => { type: Integer,
+                                       desc: "How long to wait before force-closing a connection." },
+          "encryption-algorithms" => { type: Array,
+                                       desc: "List of encryption algorithms to use when establishing a "\
+                                             "connection with a target. Supported algorithms are "\
+                                             "defined by the Ruby net-ssh library and can be viewed "\
+                                             "[here](https://github.com/net-ssh/net-ssh#supported-algorithms). "\
+                                             "All supported, non-deprecated algorithms are available by default when "\
+                                             "this option is not used. To reference all default algorithms "\
+                                             "when using this option, add 'defaults' to the list of supported "\
+                                             "algorithms." },
+          "extensions"            => { type: Array,
+                                       desc: "List of file extensions that are accepted for scripts or tasks on "\
+                                             "Windows. Scripts with these file extensions rely on the target's file "\
+                                             "type association to run. For example, if Python is installed on the "\
+                                             "system, a `.py` script runs with `python.exe`. The extensions `.ps1`, "\
+                                             "`.rb`, and `.pp` are always allowed and run via hard-coded "\
+                                             "executables." },
+          "host"                  => { type: String,
+                                       external: true,
+                                       desc: "Host name." },
+          "host-key-algorithms"   => { type: Array,
+                                       desc: "List of host key algorithms to use when establishing a connection "\
+                                             "with a target. Supported algorithms are defined by the Ruby net-ssh "\
+                                             "library "\
+                                             "([docs](https://github.com/net-ssh/net-ssh#supported-algorithms)). "\
+                                             "All supported, non-deprecated algorithms are available by default when "\
+                                             "this option is not used. To reference all default algorithms "\
+                                             "using this option, add 'defaults' to the list of supported "\
+                                             "algorithms." },
+          "host-key-check"        => { type: TrueClass,
+                                       external: true,
+                                       desc: "Whether to perform host key validation when connecting." },
+          "interpreters"          => { type: Hash,
+                                       external: true,
+                                       desc: "A map of an extension name to the absolute path of an executable, "\
+                                             "enabling you to override the shebang defined in a task executable. "\
+                                             "The extension can optionally be specified with the `.` character "\
+                                             "(`.py` and `py` both map to a task executable `task.py`) and the "\
+                                             "extension is case sensitive. When a target's name is `localhost`, "\
+                                             "Ruby tasks run with the Bolt Ruby interpreter by default." },
+          "kex-algorithms"        => { type: Array,
+                                       desc: "List of key exchange algorithms to use when establishing a "\
+                                             "connection to a target. Supported algorithms are defined by the "\
+                                             "Ruby net-ssh library "\
+                                             "([docs](https://github.com/net-ssh/net-ssh#supported-algorithms)). "\
+                                             "All supported, non-deprecated algorithms are available by default when "\
+                                             "this option is not used. To reference all default algorithms "\
+                                             "using this option, add 'defaults' to the list of supported "\
+                                             "algorithms." },
+          "load-config"           => { type: TrueClass,
+                                       desc: "Whether to load system SSH configuration." },
+          "login-shell"           => { type: String,
+                                       desc: "Which login shell Bolt should expect on the target. "\
+                                             "Supported shells are #{LOGIN_SHELLS.join(', ')}. "\
+                                             "**This option is experimental.**" },
+          "mac-algorithms"        => { type: Array,
+                                       desc: "List of message authentication code algorithms to use when "\
+                                             "establishing a connection to a target. Supported algorithms are "\
+                                             "defined by the Ruby net-ssh library "\
+                                             "([docs](https://github.com/net-ssh/net-ssh#supported-algorithms)). "\
+                                             "All supported, non-deprecated algorithms are available by default when "\
+                                             "this option is not used. To reference all default algorithms "\
+                                             "using this option, add 'defaults' to the list of supported "\
+                                             "algorithms." },
+          "password"              => { type: String,
+                                       desc: "Login password." },
+          "port"                  => { type: Integer,
+                                       external: true,
+                                       desc: "Connection port." },
+          "private-key"           => { external: true,
+                                       desc: "Either the path to the private key file to use for authentication, or "\
+                                             "a hash with the key `key-data` and the contents of the private key." },
+          "proxyjump"             => { type: String,
+                                       desc: "A jump host to proxy connections through, and an optional user to "\
+                                             "connect with." },
+          "run-as"                => { type: String,
+                                       external: true,
+                                       desc: "A different user to run commands as after login." },
+          "run-as-command"        => { type: Array,
+                                       external: true,
+                                       desc: "The command to elevate permissions. Bolt appends the user and command "\
+                                             "strings to the configured `run-as-command` before running it on the "\
+                                             "target. This command must not require an interactive password prompt, "\
+                                             "and the `sudo-password` option is ignored when `run-as-command` is "\
+                                             "specified. The `run-as-command` must be specified as an array." },
+          "script-dir"            => { type: String,
+                                       external: true,
+                                       desc: "The subdirectory of the tmpdir to use in place of a randomized "\
+                                             "subdirectory for uploading and executing temporary files on the "\
+                                             "target. It's expected that this directory already exists as a subdir "\
+                                             "of tmpdir, which is either configured or defaults to `/tmp`." },
+          "ssh-command"           => { external: true,
+                                       desc: "Command and flags to use when SSHing. This enables the external "\
+                                             "SSH transport which shells out to the specified command. "\
+                                             "**This option is experimental.**" },
+          "sudo-executable"       => { type: String,
+                                       external: true,
+                                       desc: "The executable to use when escalating to the configured `run-as` "\
+                                             "user. This is useful when you want to escalate using the configured "\
+                                             "`sudo-password`, since `run-as-command` does not use `sudo-password` "\
+                                             "or support prompting. The command executed on the target is "\
+                                             "`<sudo-executable> -S -u <user> -p custom_bolt_prompt <command>`. "\
+                                             "**This option is experimental.**" },
+          "sudo-password"         => { type: String,
+                                       external: true,
+                                       desc: "Password to use when changing users via `run-as`." },
+          "tmpdir"                => { type: String,
+                                       external: true,
+                                       desc: "The directory to upload and execute temporary files on the target." },
+          "tty"                   => { type: TrueClass,
+                                       desc: "Request a pseudo tty for the session. This option is generally "\
+                                             "only used in conjunction with the `run-as` option when the sudoers "\
+                                             "policy requires a `tty`." },
+          "user"                  => { type: String,
+                                       external: true,
+                                       desc: "Login user." }
         }.freeze
 
         DEFAULTS = {
@@ -142,10 +180,11 @@ module Bolt
                   "Unsupported login-shell #{@config['login-shell']}. Supported shells are #{LOGIN_SHELLS.join(', ')}"
           end
 
-          if (run_as_cmd = @config['run-as-command'])
-            unless run_as_cmd.all? { |n| n.is_a?(String) }
+          %w[encryption-algorithms host-key-algorithms kex-algorithms mac-algorithms run-as-command].each do |opt|
+            next unless @config.key?(opt)
+            unless @config[opt].all? { |n| n.is_a?(String) }
               raise Bolt::ValidationError,
-                    "run-as-command must be an Array of Strings, received #{run_as_cmd.class} #{run_as_cmd.inspect}"
+                    "#{opt} must be an Array of Strings, received #{@config[opt].inspect}"
             end
           end
 

data/lib/bolt/executor.rb

@@ -34,7 +34,8 @@ module Bolt
 
     def initialize(concurrency = 1,
                    analytics = Bolt::Analytics::NoopClient.new,
-                   noop = false)
+                   noop = false,
+                   modified_concurrency = false)
       # lazy-load expensive gem code
       require 'concurrent'
 
@@ -64,6 +65,9 @@ module Bolt
                 Concurrent.global_immediate_executor
               end
       @logger.debug { "Started with #{concurrency} max thread(s)" }
+
+      @concurrency = concurrency
+      @warn_concurrency = modified_concurrency
     end
 
     def transport(transport)
@@ -102,6 +106,15 @@ module Bolt
     # defined by the transport. Yields each batch, along with the corresponding
     # transport, to the block in turn and returns an array of result promises.
     def queue_execute(targets)
+      if @warn_concurrency && targets.length > @concurrency
+        @warn_concurrency = false
+        @logger.warn("The ulimit is low, which may cause file limit issues. Default concurrency has been set to "\
+                     "'#{@concurrency}' to mitigate those issues, which may cause Bolt to run slow. "\
+                     "Disable this warning by configuring ulimit using 'ulimit -n <limit>' in your shell "\
+                     "configuration, or by configuring Bolt's concurrency. "\
+                     "See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details.")
+      end
+
       targets.group_by(&:transport).flat_map do |protocol, protocol_targets|
         transport = transport(protocol)
         report_transport(transport, protocol_targets.count)

data/lib/bolt/inventory/group.rb

@@ -16,7 +16,7 @@ module Bolt
       DATA_KEYS = %w[config facts vars features plugin_hooks].freeze
       TARGET_KEYS = DATA_KEYS + %w[name alias uri]
       GROUP_KEYS = DATA_KEYS + %w[name groups targets]
-      CONFIG_KEYS = Bolt::Config::TRANSPORT_CONFIG.keys + ['transport']
+      CONFIG_KEYS = Bolt::Config::INVENTORY_CONFIG.keys
 
       def initialize(input, plugins)
         @logger = Logging.logger[self]

data/lib/bolt/inventory/inventory.rb

@@ -323,6 +323,10 @@ module Bolt
       def resources(target)
         @targets[target.name].resources
       end
+
+      def resource(target, type, title)
+        @targets[target.name].resource(type, title)
+      end
     end
   end
 end

data/lib/bolt/inventory/target.rb

@@ -90,6 +90,10 @@ module Bolt
         end
       end
 
+      def resource(type, title)
+        resources[Bolt::ResourceInstance.format_reference(type, title)]
+      end
+
       def plugin_hooks
         # Merge plugin_hooks from the config file with any defined by the group
         # or assigned dynamically to the target

data/lib/bolt/outputter.rb

@@ -8,6 +8,8 @@ module Bolt
         Bolt::Outputter::Human.new(color, verbose, trace)
       when 'json'
         Bolt::Outputter::JSON.new(color, verbose, trace)
+      when 'rainbow'
+        Bolt::Outputter::Rainbow.new(color, verbose, trace)
       when nil
         raise "Cannot use outputter before parsing."
       end
@@ -24,4 +26,5 @@ end
 
 require 'bolt/outputter/human'
 require 'bolt/outputter/json'
+require 'bolt/outputter/rainbow'
 require 'bolt/outputter/logger'

data/lib/bolt/outputter/rainbow.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'bolt/pal'
+require 'paint'
+
+module Bolt
+  class Outputter
+    class Rainbow < Bolt::Outputter::Human
+      def initialize(color, verbose, trace, stream = $stdout)
+        super
+        @line_color = 0
+        @color = 0
+        @state = :normal
+      end
+
+      # The algorithm is from lolcat (https://github.com/busyloop/lolcat)
+      # lolcat is released with WTFPL
+      def rainbow
+        red = Math.sin(0.3 * @color + 0) * 127 + 128
+        green = Math.sin(0.3 * @color + 2 * Math::PI / 3) * 127 + 128
+        blue  = Math.sin(0.3 * @color + 4 * Math::PI / 3) * 127 + 128
+        @color += 1 / 8.0
+        format("%<red>02X%<green>02X%<blue>02X", red: red, green: green, blue: blue)
+      end
+
+      def colorize(color, string)
+        if @color && @stream.isatty
+          if %i[green rainbow].include?(color)
+            a = string.chars.map do |c|
+              case @state
+              when :normal
+                if c == "\e"
+                  @state = :ansi
+                elsif c == "\n"
+                  @line_color += 1
+                  @color = @line_color
+                  c
+                else
+                  Paint[c, rainbow]
+                end
+              when :ansi
+                @state = :normal if c == 'm'
+              end
+            end
+            a.join('')
+          else
+            "\033[#{COLORS[color]}m#{string}\033[0m"
+          end
+        else
+          string
+        end
+      end
+
+      def print_summary(results, elapsed_time = nil)
+        ok_set = results.ok_set
+        unless ok_set.empty?
+          @stream.puts colorize(:rainbow, format('Successful on %<size>d target%<plural>s: %<names>s',
+                                                 size: ok_set.size,
+                                                 plural: ok_set.size == 1 ? '' : 's',
+                                                 names: ok_set.targets.map(&:safe_name).join(',')))
+        end
+
+        error_set = results.error_set
+        unless error_set.empty?
+          @stream.puts colorize(:red,
+                                format('Failed on %<size>d target%<plural>s: %<names>s',
+                                       size: error_set.size,
+                                       plural: error_set.size == 1 ? '' : 's',
+                                       names: error_set.targets.map(&:safe_name).join(',')))
+        end
+
+        total_msg = format('Ran on %<size>d target%<plural>s',
+                           size: results.size,
+                           plural: results.size == 1 ? '' : 's')
+        total_msg << " in #{duration_to_string(elapsed_time)}" unless elapsed_time.nil?
+        @stream.puts colorize(:rainbow, total_msg)
+      end
+    end
+  end
+end

data/lib/bolt/pal.rb

@@ -190,6 +190,7 @@ module Bolt
           # versions of "core" types, which are already present on the agent,
           # but may cause issues on Puppet 5 agents.
           @original_modulepath,
+          @project,
           pdb_client,
           @hiera_config,
           @max_compiles,
@@ -359,6 +360,7 @@ module Bolt
           name = param.name
           if signature_params.include?(name)
             params[name] = { 'type' => param.types.first }
+            params[name]['sensitive'] = param.types.first =~ /\ASensitive(\[.*\])?\z/ ? true : false
             params[name]['default_value'] = defaults[name] if defaults.key?(name)
             params[name]['description'] = param.text unless param.text.empty?
           else
@@ -390,6 +392,7 @@ module Bolt
                        param.type_expr
                      end
           params[name] = { 'type' => type_str }
+          params[name]['sensitive'] = param.type_expr.instance_of?(Puppet::Pops::Types::PSensitiveType)
           params[name]['default_value'] = param.value
           params[name]['description'] = param.description if param.description
         end

data/lib/bolt/project.rb

@@ -2,6 +2,7 @@
 
 require 'pathname'
 require 'bolt/pal'
+require 'bolt/config'
 
 module Bolt
   class Project
@@ -12,11 +13,14 @@ module Bolt
       "tasks" => "An array of task names to whitelist. Whitelisted plans are included in `bolt task show` output"
     }.freeze
 
-    attr_reader :path, :config_file, :inventory_file, :modulepath, :hiera_config,
-                :puppetfile, :rerunfile, :type, :resource_types
+    attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
+                :puppetfile, :rerunfile, :type, :resource_types, :warnings, :project_file
 
     def self.default_project
-      Project.new(File.join('~', '.puppetlabs', 'bolt'), 'user')
+      create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user')
+    # If homedir isn't defined use the system config path
+    rescue ArgumentError
+      create_project(Bolt::Config.system_path, 'system')
     end
 
     # Search recursively up the directory hierarchy for the Project. Look for a
@@ -26,9 +30,9 @@ module Bolt
     def self.find_boltdir(dir)
       dir = Pathname.new(dir)
       if (dir + BOLTDIR_NAME).directory?
-        new(dir + BOLTDIR_NAME, 'embedded')
+        create_project(dir + BOLTDIR_NAME, 'embedded')
       elsif (dir + 'bolt.yaml').file? || (dir + 'bolt-project.yaml').file?
-        new(dir, 'local')
+        create_project(dir, 'local')
       elsif dir.root?
         default_project
       else
@@ -36,9 +40,25 @@ module Bolt
       end
     end
 
-    def initialize(path, type = 'option')
+    def self.create_project(path, type = 'option')
+      fullpath = Pathname.new(path).expand_path
+      project_file = File.join(fullpath, 'bolt-project.yaml')
+      data = Bolt::Util.read_optional_yaml_hash(File.expand_path(project_file), 'project')
+      new(data, path, type)
+    end
+
+    def initialize(raw_data, path, type = 'option')
       @path = Pathname.new(path).expand_path
-      @config_file = @path + 'bolt.yaml'
+      @project_file = @path + 'bolt-project.yaml'
+
+      @warnings = []
+      if (@path + 'bolt.yaml').file? && project_file?
+        msg = "Project-level configuration in bolt.yaml is deprecated if using bolt-project.yaml. "\
+          "Transport config should be set in inventory.yaml, all other config should be set in "\
+          "bolt-project.yaml."
+        @warnings << { msg: msg }
+      end
+
       @inventory_file = @path + 'inventory.yaml'
       @modulepath = [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
       @hiera_config = @path + 'hiera.yaml'
@@ -47,9 +67,26 @@ module Bolt
       @resource_types = @path + '.resource_types'
       @type = type
 
-      @project_file = @path + 'bolt-project.yaml'
-      @data = Bolt::Util.read_optional_yaml_hash(File.expand_path(@project_file), 'project') || {}
-      validate if load_as_module?
+      tc = Bolt::Config::INVENTORY_CONFIG.keys & raw_data.keys
+      if tc.any?
+        msg = "Transport configuration isn't supported in bolt-project.yaml. Ignoring keys #{tc}"
+        @warnings << { msg: msg }
+      end
+
+      @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_CONFIG.keys.include?(k) }
+
+      # Once bolt.yaml deprecation is removed, this attribute should be removed
+      # and replaced with .project_file in lib/bolt/config.rb
+      @config_file = if (Bolt::Config::OPTIONS.keys & @data.keys).any?
+                       if (@path + 'bolt.yaml').file?
+                         msg = "bolt-project.yaml contains valid config keys, bolt.yaml will be ignored"
+                         @warnings << { msg: msg }
+                       end
+                       @project_file
+                     else
+                       @path + 'bolt.yaml'
+                     end
+      validate if project_file?
     end
 
     def to_s
@@ -67,7 +104,7 @@ module Bolt
     end
     alias == eql?
 
-    def load_as_module?
+    def project_file?
       @project_file.file?
     end