bolt 2.17.0 → 2.22.0

data/lib/bolt/config.rb

@@ -19,7 +19,7 @@ module Bolt
   class Config
     include Bolt::Config::Options
 
-    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency
+    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency, :deprecations
 
     BOLT_CONFIG_NAME = 'bolt.yaml'
     BOLT_DEFAULTS_NAME = 'bolt-defaults.yaml'
@@ -28,7 +28,7 @@ module Bolt
     DEFAULT_DEFAULT_CONCURRENCY = 100
 
     def self.default
-      new(Bolt::Project.create_project('.'), {})
+      new(Bolt::Project.default_project, {})
     end
 
     def self.from_project(project, overrides = {})
@@ -41,7 +41,8 @@ module Bolt
       data = load_defaults(project).push(
         filepath: project.config_file,
         data: conf,
-        warnings: []
+        warnings: [],
+        deprecations: []
       )
 
       new(project, data, overrides)
@@ -59,7 +60,8 @@ module Bolt
       data = load_defaults(project).push(
         filepath: project.config_file,
         data: conf,
-        warnings: []
+        warnings: [],
+        deprecations: []
       )
 
       new(project, data, overrides)
@@ -121,13 +123,26 @@ module Bolt
         )
       end
 
-      # Move data under transport-config to top-level so it can be easily merged with
-      # config from other sources.
+      # Move data under inventory-config to top-level so it can be easily merged with
+      # config from other sources. Error early if inventory-config is not a hash or
+      # has a plugin reference.
       if data.key?('inventory-config')
+        unless data['inventory-config'].is_a?(Hash)
+          raise Bolt::ValidationError,
+                "Option 'inventory-config' must be of type Hash, received #{data['inventory-config']} "\
+                "#{data['inventory-config']} (file: #{filepath})"
+        end
+
+        if data['inventory-config'].key?('_plugin')
+          raise Bolt::ValidationError,
+                "Found unsupported key '_plugin' for option 'inventory-config'; supported keys are "\
+                "'#{INVENTORY_OPTIONS.keys.join("', '")}' (file: #{filepath})"
+        end
+
         data = data.merge(data.delete('inventory-config'))
       end
 
-      { filepath: filepath, data: data, warnings: warnings }
+      { filepath: filepath, data: data, warnings: warnings, deprecations: [] }
     end
 
     # Loads a 'bolt.yaml' file, the legacy configuration file. There's no special munging needed
@@ -135,10 +150,11 @@ module Bolt
     def self.load_bolt_yaml(dir)
       filepath = dir + BOLT_CONFIG_NAME
       data     = Bolt::Util.read_yaml_hash(filepath, 'config')
-      warnings = [msg: "Configuration file #{filepath} is deprecated and will be removed in a future version "\
-                        "of Bolt. Use '#{dir + BOLT_DEFAULTS_NAME}' instead."]
+      deprecations = [{ type: 'Using bolt.yaml for system configuration',
+                        msg: "Configuration file #{filepath} is deprecated and will be removed in a future version "\
+                        "of Bolt. Use '#{dir + BOLT_DEFAULTS_NAME}' instead." }]
 
-      { filepath: filepath, data: data, warnings: warnings }
+      { filepath: filepath, data: data, warnings: [], deprecations: deprecations }
     end
 
     def self.load_defaults(project)
@@ -169,12 +185,16 @@ module Bolt
 
     def initialize(project, config_data, overrides = {})
       unless config_data.is_a?(Array)
-        config_data = [{ filepath: project.config_file, data: config_data, warnings: [] }]
+        config_data = [{ filepath: project.config_file,
+                         data: config_data,
+                         warnings: [],
+                         deprecations: [] }]
       end
 
       @logger       = Logging.logger[self]
       @project      = project
       @warnings     = @project.warnings.dup
+      @deprecations = @project.deprecations.dup
       @transports   = {}
       @config_files = []
 
@@ -195,6 +215,7 @@ module Bolt
 
       loaded_data = config_data.each_with_object([]) do |data, acc|
         @warnings.concat(data[:warnings]) if data[:warnings].any?
+        @deprecations.concat(data[:deprecations]) if data[:deprecations].any?
 
         if data[:data].any?
           @config_files.push(data[:filepath])
@@ -365,7 +386,7 @@ module Bolt
         raise Bolt::ValidationError, "Compilation is CPU-intensive, set concurrency less than #{compile_limit}"
       end
 
-      if (format == 'rainbow' && Bolt::Util.windows?) || !(%w[human json rainbow].include? format)
+      unless %w[human json rainbow].include? format
         raise Bolt::ValidationError, "Unsupported format: '#{format}'"
       end
 
@@ -478,7 +499,7 @@ module Bolt
     end
 
     def matching_paths(paths)
-      [*paths].map { |p| Dir.glob([p, casefold(p)]) }.flatten.uniq.reject { |p| [*paths].include?(p) }
+      Array(paths).map { |p| Dir.glob([p, casefold(p)]) }.flatten.uniq.reject { |p| Array(paths).include?(p) }
     end
 
     private def casefold(path)

data/lib/bolt/config/options.rb

@@ -175,7 +175,7 @@ module Bolt
         "log" => {
           description: "A map of configuration for the logfile output. Under `log`, you can configure log options "\
                        "for `console` and add configuration for individual log files, such as "\
-                       "~/.puppetlabs/bolt/debug.log`. Individual log files must be valid filepaths. If the log "\
+                       "`~/.puppetlabs/bolt/debug.log`. Individual log files must be valid filepaths. If the log "\
                        "file does not exist, then Bolt will create it before logging information.",
           type: Hash,
           properties: {
@@ -275,11 +275,25 @@ module Bolt
               type: String,
               _example: "/etc/puppetlabs/puppet/ssl/certs/my-host.example.com.pem"
             },
+            "connect_timeout" => {
+              description: "How long to wait in seconds when establishing connections with PuppetDB.",
+              type: Integer,
+              minimum: 1,
+              _default: 60,
+              _example: 120
+            },
             "key" => {
               description: "The private key for the certificate.",
               type: String,
               _example: "/etc/puppetlabs/puppet/ssl/private_keys/my-host.example.com.pem"
             },
+            "read_timeout" => {
+              description: "How long to wait in seconds for a response from PuppetDB.",
+              type: Integer,
+              minimum: 1,
+              _default: 60,
+              _example: 120
+            },
             "server_urls" => {
               description: "An array containing the PuppetDB host to connect to. Include the protocol `https` "\
                            "and the port, which is usually `8081`. For example, "\
@@ -440,6 +454,7 @@ module Bolt
         concurrency
         format
         inventory-config
+        log
         plugin_hooks
         plugins
         puppetdb

data/lib/bolt/config/transport/options.rb

@@ -108,17 +108,16 @@ module Bolt
           },
           "copy-command" => {
             type: [Array, String],
-            description: "The command to use when copying files using ssh-command. Bolt runs `<copy-command> <src> "\
+            description: "The command to use when copying files using native SSH. Bolt runs `<copy-command> <src> "\
                          "<dest>`. This option is used when you need support for features or algorithms that are not "\
                          "supported by the net-ssh Ruby library. **This option is experimental.** You can read more "\
-                         "about this option in [External SSH "\
-                         "transport](experimental_features.md#external-ssh-transport).",
+                         "about this option in [Native SSH transport](experimental_features.md#native-ssh-transport).",
             items: {
               type: String
             },
             _plugin: true,
-            _default: "scp -r",
-            _example: "scp -r -F ~/ssh-config/myconf"
+            _default: %w[scp -r],
+            _example: %w[scp -r -F ~/ssh-config/myconf]
           },
           "disconnect-timeout" => {
             type: Integer,
@@ -270,6 +269,13 @@ module Bolt
             _plugin: true,
             _example: %w[defaults hmac-md5]
           },
+          "native-ssh" => {
+            type: [TrueClass, FalseClass],
+            description: "This enables the native SSH transport, which shells out to SSH instead of using the "\
+                         "net-ssh Ruby library",
+            _default: false,
+            _example: true
+          },
           "password" => {
             type: String,
             description: "The password to use to login.",
@@ -368,16 +374,16 @@ module Bolt
           },
           "ssh-command" => {
             type: [Array, String],
-            description: "The command and flags to use when SSHing. This enables the external SSH transport, which "\
-                         "shells out to the specified command. This option is used when you need support for "\
+            description: "The command and flags to use when SSHing. This option is used when you need support for "\
                          "features or algorithms that are not supported by the net-ssh Ruby library. **This option "\
-                         "is experimental.** You can read more about this  option in [External SSH "\
-                         "transport](experimental_features.md#external-ssh-transport).",
+                         "is experimental.** You can read more about this  option in [Native SSH "\
+                         "transport](experimental_features.md#native-ssh-transport).",
             items: {
               type: String
             },
             _plugin: true,
-            _example: "ssh"
+            _default: 'ssh',
+            _example: %w[ssh -o Ciphers=chacha20-poly1305@openssh.com]
           },
           "ssl" => {
             type: [TrueClass, FalseClass],

data/lib/bolt/config/transport/ssh.rb

@@ -32,13 +32,14 @@ module Bolt
           user
         ].concat(RUN_AS_OPTIONS).sort.freeze
 
-        # Options available when using the external ssh transport
-        EXTERNAL_OPTIONS = %w[
+        # Options available when using the native ssh transport
+        NATIVE_OPTIONS = %w[
           cleanup
           copy-command
           host
           host-key-check
           interpreters
+          native-ssh
           port
           private-key
           script-dir
@@ -56,17 +57,30 @@ module Bolt
           "tty"                => false
         }.freeze
 
-        # The set of options available for the ssh and external ssh transports overlap, so we
+        # The set of options available for the ssh and native ssh transports overlap, so we
         # need to check which transport is used before fully initializing, otherwise options
         # may not be filtered correctly.
         def initialize(data = {}, project = nil)
           assert_hash_or_config(data)
-          @external = true if data['ssh-command']
+          @native = true if data['native-ssh']
           super(data, project)
         end
 
+        # This method is used to filter CLI options in the Config class. This
+        # should include `ssh-command` so that we can later warn if the option
+        # is present without `native-ssh`
+        def self.options
+          %w[ssh-command native-ssh].concat(OPTIONS)
+        end
+
         private def filter(unfiltered)
-          @external ? unfiltered.slice(*EXTERNAL_OPTIONS) : unfiltered.slice(*OPTIONS)
+          # Because we filter before merging config together it's impossible to
+          # know whether both ssh-command *and* native-ssh will be specified
+          # unless they are both in the filter. However, we can't add
+          # ssh-command to OPTIONS since that's used for documenting available
+          # options. This makes it so that ssh-command is preserved so we can
+          # warn once all config is resolved if native-ssh isn't set.
+          @native ? unfiltered.slice(*NATIVE_OPTIONS) : unfiltered.slice(*self.class.options)
         end
 
         private def validate
@@ -83,11 +97,11 @@ module Bolt
               @config['private-key'] = File.expand_path(key_opt, @project)
 
               # We have an explicit test for this to only warn if using net-ssh transport
-              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['ssh-command']
+              Bolt::Util.validate_file('ssh key', @config['private-key']) if @config['native-ssh']
             end
 
-            if key_opt.instance_of?(Hash) && @config['ssh-command']
-              raise Bolt::ValidationError, 'private-key must be a filepath when using ssh-command'
+            if key_opt.instance_of?(Hash) && @config['native-ssh']
+              raise Bolt::ValidationError, 'private-key must be a filepath when using native-ssh'
             end
           end
 
@@ -117,8 +131,8 @@ module Bolt
             end
           end
 
-          if @config['ssh-command'] && !@config['load-config']
-            msg = 'Cannot use external SSH transport with load-config set to false'
+          if @config['native-ssh'] && !@config['load-config']
+            msg = 'Cannot use native SSH transport with load-config set to false'
             raise Bolt::ValidationError, msg
           end
         end

data/lib/bolt/executor.rb

@@ -320,6 +320,27 @@ module Bolt
       end
     end
 
+    def download_file(targets, source, destination, options = {})
+      description = options.fetch(:description, "file download from #{source} to #{destination}")
+
+      begin
+        FileUtils.mkdir_p(destination)
+      rescue Errno::EEXIST => e
+        message = "#{e.message}; unable to create destination directory #{destination}"
+        raise Bolt::Error.new(message, 'bolt/file-exist-error')
+      end
+
+      log_action(description, targets) do
+        options[:run_as] = run_as if run_as && !options.key?(:run_as)
+
+        batch_execute(targets) do |transport, batch|
+          with_node_logging("Downloading file #{source} to #{destination}", batch) do
+            transport.batch_download(batch, source, destination, options, &method(:publish_event))
+          end
+        end
+      end
+    end
+
     def run_plan(scope, plan, params)
       plan.call_by_name_with_scope(scope, params, true)
     end

data/lib/bolt/inventory/group.rb

@@ -50,10 +50,11 @@ module Bolt
           # or it could be a name/alias of a target defined in another group.
           # We can't tell the difference until all groups have been resolved,
           # so we store the string on its own here and process it later.
-          if target.is_a?(String)
+          case target
+          when String
             @string_targets << target
           # Handle plugins at this level so that lookups cannot trigger recursive lookups
-          elsif target.is_a?(Hash)
+          when Hash
             add_target_definition(target)
           else
             raise ValidationError.new("Target entry must be a String or Hash, not #{target.class}", @name)

data/lib/bolt/inventory/inventory.rb

@@ -109,11 +109,12 @@ module Bolt
       private :resolve_name
 
       def expand_targets(targets)
-        if targets.is_a? Bolt::Target
+        case targets
+        when Bolt::Target
           targets
-        elsif targets.is_a? Array
+        when Array
           targets.map { |tish| expand_targets(tish) }
-        elsif targets.is_a? String
+        when String
           # Expand a comma-separated list
           targets.split(/[[:space:],]+/).reject(&:empty?).map do |name|
             ts = resolve_name(name)

data/lib/bolt/logger.rb

@@ -15,6 +15,7 @@ module Bolt
       return if Logging.initialized?
 
       Logging.init :debug, :info, :notice, :warn, :error, :fatal, :any
+      @mutex = Mutex.new
 
       Logging.color_scheme(
         'bolt',
@@ -66,6 +67,10 @@ module Bolt
       end
     end
 
+    def self.analytics=(analytics)
+      @analytics = analytics
+    end
+
     def self.console_layout(color)
       color_scheme = :bolt if color
       Logging.layouts.pattern(
@@ -102,5 +107,21 @@ module Bolt
     def self.reset_logging
       Logging.reset
     end
+
+    def self.warn_once(type, msg)
+      @mutex.synchronize {
+        @warnings ||= []
+        @logger ||= Logging.logger[self]
+        unless @warnings.include?(type)
+          @logger.warn(msg)
+          @warnings << type
+        end
+      }
+    end
+
+    def self.deprecation_warning(type, msg)
+      @analytics&.event('Warn', 'deprecation', label: type)
+      warn_once(type, msg)
+    end
   end
 end

data/lib/bolt/module.rb

@@ -3,7 +3,8 @@
 # Is this Bolt::Pobs::Module?
 module Bolt
   class Module
-    MODULE_NAME_REGEX = /\A[a-z][a-z0-9_]*\z/.freeze
+    CONTENT_NAME_REGEX = /\A[a-z][a-z0-9_]*(::[a-z][a-z0-9_]*)*\z/.freeze
+    MODULE_NAME_REGEX  = /\A[a-z][a-z0-9_]*\z/.freeze
 
     def self.discover(modulepath)
       modulepath.each_with_object({}) do |path, mods|

data/lib/bolt/outputter/rainbow.rb

@@ -8,6 +8,12 @@ module Bolt
       def initialize(color, verbose, trace, stream = $stdout)
         begin
           require 'paint'
+          if Bolt::Util.windows?
+            # the Paint gem thinks that windows does not support ansi colors
+            # but windows 10 or later does
+            # we can display colors if we force mode to TRUE_COLOR
+            Paint.mode = 0xFFFFFF
+          end
         rescue LoadError
           raise "The 'paint' gem is required to use the rainbow outputter."
         end
@@ -33,9 +39,10 @@ module Bolt
             a = string.chars.map do |c|
               case @state
               when :normal
-                if c == "\e"
+                case c
+                when "\e"
                   @state = :ansi
-                elsif c == "\n"
+                when "\n"
                   @line_color += 1
                   @color = @line_color
                   c

data/lib/bolt/pal.rb

@@ -150,7 +150,12 @@ module Bolt
       r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
         bolt_project = @project if @project&.name
+        # Puppet currently won't receive the project unless it is a named project. Since
+        # the download_file plan function needs access to the project path, add it to the
+        # context.
+        bolt_project_data = @project
         Puppet.override(bolt_project: bolt_project,
+                        bolt_project_data: bolt_project_data,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
           pal.with_script_compiler do |compiler|
             alias_types(compiler)
@@ -279,9 +284,10 @@ module Bolt
 
     def parse_params(type, object_name, params)
       in_bolt_compiler do |compiler|
-        if type == 'task'
+        case type
+        when 'task'
           param_spec = compiler.task_signature(object_name)&.task_hash&.dig('parameters')
-        elsif type == 'plan'
+        when 'plan'
           plan = compiler.plan_signature(object_name)
           param_spec = plan.params_type.elements&.each_with_object({}) { |t, h| h[t.name] = t.value_type } if plan
         end