bolt 2.37.0 → 2.44.0

data/lib/bolt/config/transport/orch.rb

@@ -12,6 +12,7 @@ module Bolt
           host
           job-poll-interval
           job-poll-timeout
+          read-timeout
           service-url
           task-environment
           token-file

data/lib/bolt/config/transport/ssh.rb

@@ -111,11 +111,6 @@ module Bolt
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
 
-          if @config['login-shell'] && !LOGIN_SHELLS.include?(@config['login-shell'])
-            raise Bolt::ValidationError,
-                  "Unsupported login-shell #{@config['login-shell']}. Supported shells are #{LOGIN_SHELLS.join(', ')}"
-          end
-
           if @config['login-shell'] == 'powershell'
             %w[tty run-as].each do |key|
               if @config[key]

data/lib/bolt/executor.rb

@@ -100,6 +100,8 @@ module Bolt
         # that type of event, publish the event
         next unless types.nil? || types.include?(event[:type])
         @publisher.post(subscriber) do |sub|
+          # Wait for user to input to prompt before printing anything
+          sleep(0.1) while @prompting
           sub.handle_event(event)
         end
       end
@@ -119,11 +121,12 @@ module Bolt
     def queue_execute(targets)
       if @warn_concurrency && targets.length > @concurrency
         @warn_concurrency = false
-        @logger.warn("The ulimit is low, which may cause file limit issues. Default concurrency has been set to "\
-                     "'#{@concurrency}' to mitigate those issues, which may cause Bolt to run slow. "\
-                     "Disable this warning by configuring ulimit using 'ulimit -n <limit>' in your shell "\
-                     "configuration, or by configuring Bolt's concurrency. "\
-                     "See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details.")
+        msg = "The ulimit is low, which may cause file limit issues. Default concurrency has been set to "\
+              "'#{@concurrency}' to mitigate those issues, which may cause Bolt to run slow. "\
+              "Disable this warning by configuring ulimit using 'ulimit -n <limit>' in your shell "\
+              "configuration, or by configuring Bolt's concurrency. "\
+              "See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+        Bolt::Logger.warn("low_ulimit", msg)
       end
 
       targets.group_by(&:transport).flat_map do |protocol, protocol_targets|
@@ -258,7 +261,9 @@ module Bolt
 
     def with_node_logging(description, batch, log_level = :info)
       @logger.send(log_level, "#{description} on #{batch.map(&:safe_name)}")
+      publish_event(type: :start_spin)
       result = yield
+      publish_event(type: :stop_spin)
       @logger.send(log_level, result.to_json)
       result
     end
@@ -410,6 +415,7 @@ module Bolt
       subscribe(self, [:node_result])
       results = Array.new(skein.length)
       @in_parallel = true
+      publish_event(type: :stop_spin)
 
       until skein.empty?
         @thread_completed = false
@@ -417,6 +423,7 @@ module Bolt
 
         skein.each do |yarn|
           if yarn.alive?
+            publish_event(type: :stop_spin)
             r = yarn.resume
           else
             results[yarn.index] = yarn.value
@@ -428,6 +435,7 @@ module Bolt
         sleep(0.1) until @thread_completed || skein.empty?
       end
 
+      publish_event(type: :stop_spin)
       @in_parallel = false
       unsubscribe(self, [:node_result])
       results
@@ -469,6 +477,7 @@ module Bolt
     end
 
     def prompt(prompt, options)
+      @prompting = true
       unless $stdin.tty?
         raise Bolt::Error.new('STDIN is not a tty, unable to prompt', 'bolt/no-tty-error')
       end
@@ -480,6 +489,7 @@ module Bolt
               else
                 $stdin.gets.to_s.chomp
               end
+      @prompting = false
 
       $stderr.puts if options[:sensitive]
 

data/lib/bolt/inventory.rb

@@ -4,13 +4,17 @@ require 'set'
 require 'bolt/config'
 require 'bolt/inventory/group'
 require 'bolt/inventory/inventory'
+require 'bolt/inventory/options'
 require 'bolt/target'
 require 'bolt/util'
 require 'bolt/plugin'
+require 'bolt/validator'
 require 'yaml'
 
 module Bolt
   class Inventory
+    include Bolt::Inventory::Options
+
     ENVIRONMENT_VAR = 'BOLT_INVENTORY'
 
     class ValidationError < Bolt::Error
@@ -45,11 +49,26 @@ module Bolt
       end
     end
 
+    # Builds the schema used by the validator.
+    #
+    def self.schema
+      schema = {
+        type:        Hash,
+        properties:  OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: DEFINITIONS,
+        _plugin:     true
+      }
+
+      schema[:definitions]['config'][:properties] = Bolt::Config.transport_definitions
+      schema
+    end
+
     def self.from_config(config, plugins)
       logger = Bolt::Logger.logger(self)
 
       if ENV.include?(ENVIRONMENT_VAR)
         begin
+          source = ENVIRONMENT_VAR
           data = YAML.safe_load(ENV[ENVIRONMENT_VAR])
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}" unless data.is_a?(Hash)
           logger.debug("Loaded inventory from environment variable #{ENVIRONMENT_VAR}")
@@ -57,9 +76,11 @@ module Bolt
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
       elsif config.inventoryfile
+        source = config.inventoryfile
         data = Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
         logger.debug("Loaded inventory from #{config.inventoryfile}")
       else
+        source = config.default_inventoryfile
         data = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
 
         if config.default_inventoryfile.exist?
@@ -74,6 +95,11 @@ module Bolt
         t.resolve(plugins) unless t.resolved?
       end
 
+      Bolt::Validator.new.tap do |validator|
+        validator.validate(data, schema, source)
+        validator.warnings.each { |warning| Bolt::Logger.warn(warning[:id], warning[:msg]) }
+      end
+
       inventory = create_version(data, config.transport, config.transports, plugins)
       inventory.validate
       inventory

data/lib/bolt/inventory/group.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'bolt/config/options'
 require 'bolt/inventory/group'
 require 'bolt/inventory/inventory'
 require 'bolt/inventory/target'
@@ -18,12 +19,23 @@ module Bolt
       GROUP_KEYS = DATA_KEYS + %w[name groups targets]
       CONFIG_KEYS = Bolt::Config::INVENTORY_OPTIONS.keys
 
-      def initialize(input, plugins)
+      def initialize(input, plugins, all_group: false)
         @logger = Bolt::Logger.logger(self)
         @plugins = plugins
 
         input = @plugins.resolve_top_level_references(input) if @plugins.reference?(input)
 
+        if all_group
+          if input.key?('name') && input['name'] != 'all'
+            Bolt::Logger.warn(
+              "top_level_group_name",
+              "Top-level group '#{input['name']}' cannot specify a name, using 'all' instead."
+            )
+          end
+
+          input = input.merge('name' => 'all')
+        end
+
         raise ValidationError.new("Group does not have a name", nil) unless input.key?('name')
 
         @name = @plugins.resolve_references(input['name'])
@@ -125,7 +137,7 @@ module Bolt
 
         unless (unexpected_keys = target.keys - TARGET_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in target #{t_name}"
-          @logger.warn(msg)
+          Bolt::Logger.warn("unknown_target_keys", msg)
         end
 
         validate_data_keys(target, t_name)
@@ -252,15 +264,7 @@ module Bolt
 
         unless (unexpected_keys = input.keys - GROUP_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in group #{@name}"
-          @logger.warn(msg)
-        end
-
-        Bolt::Util.walk_keys(input) do |key|
-          if @plugins.reference?(key)
-            raise ValidationError.new("Group keys cannot be specified as _plugin references", @name)
-          else
-            key
-          end
+          Bolt::Logger.warn("unknown_group_keys", msg)
         end
       end
 
@@ -323,7 +327,26 @@ module Bolt
           'features' => @plugins.resolve_references(data.fetch('features', [])),
           'plugin_hooks' => @plugins.resolve_references(data.fetch('plugin_hooks', {}))
         }
+
         validate_data_keys(result, target)
+
+        Bolt::Config::Options::TRANSPORT_CONFIG.each_key do |transport|
+          next unless result['config'].key?(transport)
+          transport_config = result['config'][transport]
+          next unless transport_config.is_a?(Hash)
+          transport_config = Bolt::Util.postwalk_vals(transport_config) do |val|
+            if val.is_a?(Hash)
+              val = val.compact
+              val = nil if val.empty?
+            end
+            val
+          end
+          # the transport config is user-specified data so we
+          # still want to preserve it even if it exclusively
+          # contains nil-resolved keys
+          result['config'][transport] = transport_config || {}
+        end
+
         result['features'] = Set.new(result['features'].flatten)
         result
       end
@@ -348,7 +371,7 @@ module Bolt
             msg = +"Found unexpected key(s) #{unexpected_keys.join(', ')} in config for"
             msg << " target #{target} in" if target
             msg << " group #{@name}"
-            @logger.warn(msg)
+            Bolt::Logger.warn("unknown_config_keys", msg)
           end
         end
       end

data/lib/bolt/inventory/inventory.rb

@@ -21,7 +21,7 @@ module Bolt
         @transport    = transport
         @config       = transports
         @plugins      = plugins
-        @groups       = Group.new(@data.merge('name' => 'all'), plugins)
+        @groups       = Group.new(@data, plugins, all_group: true)
         @group_lookup = {}
         @targets      = {}
 

data/lib/bolt/inventory/options.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'bolt/config/options'
+
+module Bolt
+  class Inventory
+    module Options
+      # Top-level options available in the inventory.
+      OPTIONS = %w[
+        config
+        facts
+        features
+        groups
+        targets
+        vars
+      ].freeze
+
+      # Definitions used to validate the data.
+      # https://github.com/puppetlabs/bolt/blob/main/schemas/README.md
+      DEFINITIONS = {
+        "alias" => {
+          description: "A unique alias to refer to the target. Aliases cannot conflict "\
+                       "with the name of a group, the name of a target, or another alias.",
+          type: [String, Array],
+          uniqueItems: true,
+          items: {
+            type: String,
+            _plugin: true
+          },
+          _plugin: true
+        },
+        "config" => {
+          description: "A map of configuration options.",
+          type: Hash,
+          # These properties are populated as part of Bolt::Inventory.schema
+          properties: {},
+          _plugin: true
+        },
+        "facts" => {
+          description: "A map of system information, also known as facts, for the target.",
+          type: Hash,
+          _plugin: true
+        },
+        "features" => {
+          description: "A list of available features for the target.",
+          type: Array,
+          uniqueItems: true,
+          items: {
+            type: String,
+            _plugin: true
+          },
+          _plugin: true
+        },
+        "groups" => {
+          description: "A list of groups and their associated configuration.",
+          type: Array,
+          items: {
+            type: Hash,
+            required: ["name"],
+            properties: {
+              "config"       => { _ref: "config" },
+              "facts"        => { _ref: "facts" },
+              "features"     => { _ref: "features" },
+              "groups"       => { _ref: "groups" },
+              "name"         => { _ref: "name" },
+              "plugin_hooks" => { _ref: "plugin_hooks" },
+              "targets"      => { _ref: "targets" },
+              "vars"         => { _ref: "vars" }
+            },
+            _plugin: true
+          },
+          _plugin: true
+        },
+        "name" => {
+          description: "A human-readable name to refer to the group or target. Names "\
+                       "cannot conflict with the name of a group, the name of a target, "\
+                       "or the alias of a target. A name is required for a group and is "\
+                       "required for a target unless the uri option is set.",
+          type: String,
+          _plugin: true
+        },
+        "plugin_hooks" => {
+          description: "Configuration for the Puppet library plugin used to install the "\
+                       "Puppet agent on the target. For more information, see "\
+                       "https://pup.pt/bolt-plugin-hooks",
+          type: Hash,
+          properties: {
+            "puppet_library" => {
+              description: "Configuration for the Puppet library plugin.",
+              type: Hash,
+              _plugin: true
+            }
+          },
+          _plugin: true
+        },
+        "targets" => {
+          description: "A list of targets and their associated configuration.",
+          type: Array,
+          items: {
+            type: [String, Hash],
+            properties: {
+              "alias"        => { _ref: "alias" },
+              "config"       => { _ref: "config" },
+              "facts"        => { _ref: "facts" },
+              "features"     => { _ref: "features" },
+              "name"         => { _ref: "name" },
+              "plugin_hooks" => { _ref: "plugin_hooks" },
+              "uri"          => { _ref: "uri" },
+              "vars"         => { _ref: "vars" }
+            },
+            _plugin: true
+          },
+          _plugin: true
+        },
+        "uri" => {
+          description: "The URI of the target. This option is required unless the name "\
+                       "option is set.",
+          type: String,
+          format: "uri",
+          _plugin: true
+        },
+        "vars" => {
+          description: "A map of variables for the group or target.",
+          type: Hash,
+          _plugin: true
+        }
+      }.freeze
+    end
+  end
+end

data/lib/bolt/inventory/target.rb

@@ -31,7 +31,8 @@ module Bolt
         end
 
         if @name == 'localhost'
-          target_data = localhost_defaults(target_data)
+          default = { 'config' => { 'transport' => 'local' } }
+          target_data = Bolt::Util.deep_merge(default, target_data)
         end
 
         @config = target_data['config'] || {}
@@ -49,18 +50,16 @@ module Bolt
         validate
       end
 
-      def localhost_defaults(data)
+      def set_local_defaults
+        return if @set_local_default
         defaults = {
-          'config' => {
-            'transport' => 'local',
-            'local' => { 'interpreters' => { '.rb' => RbConfig.ruby } }
-          },
-          'features' => ['puppet-agent']
+          'local' => { 'interpreters' => { '.rb' => RbConfig.ruby } }
         }
-        data = Bolt::Util.deep_merge(defaults, data)
-        # If features is an empty array deep_merge won't add the puppet-agent
-        data['features'] += ['puppet-agent'] if data['features'].empty?
-        data
+        old_config = @config
+        @config = Bolt::Util.deep_merge(defaults, @config)
+        invalidate_config_cache! if old_config != @config
+        set_feature('puppet-agent')
+        @set_local_default = true
       end
 
       # rubocop:disable Naming/AccessorMethodName

data/lib/bolt/logger.rb

@@ -5,8 +5,14 @@ require 'logging'
 module Bolt
   module Logger
     LEVELS = %w[trace debug info notice warn error fatal].freeze
-    @mutex = Mutex.new
-    @warnings = Set.new
+
+    # This module is treated as a global singleton so that multiple classes
+    # in Bolt can log warnings with IDs. Access to the following variables
+    # are controlled by a mutex.
+    @mutex            = Mutex.new
+    @warnings         = Set.new
+    @disable_warnings = Set.new
+    @message_queue    = []
 
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
@@ -36,7 +42,7 @@ module Bolt
       end
     end
 
-    def self.configure(destinations, color)
+    def self.configure(destinations, color, disable_warnings = nil)
       root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
@@ -73,6 +79,16 @@ module Bolt
 
         appender.level = params[:level] if params[:level]
       end
+
+      # Set the list of disabled warnings and mark the logger as configured.
+      # Log all messages in the message queue and flush the queue.
+      if disable_warnings
+        @mutex.synchronize { @disable_warnings = disable_warnings }
+      end
+    end
+
+    def self.configured?
+      Logging.logger[:root].appenders.any?
     end
 
     # A helper to ensure the Logging library is always initialized with our
@@ -123,18 +139,106 @@ module Bolt
       Logging.reset
     end
 
-    def self.warn_once(type, msg)
+    # The following methods are used in place of the Logging.logger
+    # methods of the same name when logging warning messages or logging
+    # any messages prior to the logger being configured. If the logger
+    # is not configured when any of these methods are called, the message
+    # will be added to a queue, otherwise they are logged immediately.
+    # The message queue is flushed by calling #flush_queue, which is
+    # called from Bolt::CLI after configuring the logger.
+    #
+    def self.warn(id, msg)
+      log(type: :warn, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.warn_once(id, msg)
+      log(type: :warn_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate(id, msg)
+      log(type: :deprecate, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate_once(id, msg)
+      log(type: :deprecate_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.debug(msg)
+      log(type: :debug, msg: msg)
+    end
+
+    def self.info(msg)
+      log(type: :info, msg: msg)
+    end
+
+    # Logs a message. If the logger has not been configured, this will queue
+    # the message to be logged later. Once the logger is configured, the
+    # queue will be flushed of all messages and new messages will be logged
+    # immediately.
+    #
+    # Logging with this method is controlled by a mutex, as the Bolt::Logger
+    # module is treated as a global singleton to allow multiple classes
+    # access to its methods.
+    #
+    private_class_method def self.log(type:, msg:, id: nil)
+      @mutex.synchronize do
+        if configured?
+          log_message(type: type, msg: msg, id: id)
+        else
+          @message_queue << { type: type, msg: msg, id: id }
+        end
+      end
+    end
+
+    # Logs all messages in the message queue and then flushes the queue.
+    #
+    def self.flush_queue
       @mutex.synchronize do
-        @logger ||= Bolt::Logger.logger(self)
-        if @warnings.add?(type)
-          @logger.warn(msg)
+        @message_queue.each do |message|
+          log_message(message)
         end
+
+        @message_queue.clear
+      end
+    end
+
+    # Handles the actual logging of a message.
+    #
+    private_class_method def self.log_message(type:, msg:, id: nil)
+      case type
+      when :warn
+        do_warn(msg, id)
+      when :warn_once
+        do_warn_once(msg, id)
+      when :deprecate
+        do_deprecate(msg, id)
+      when :deprecate_once
+        do_deprecate_once(msg, id)
+      else
+        logger(self).send(type, msg)
       end
     end
 
-    def self.deprecation_warning(type, msg)
-      @analytics&.event('Warn', 'deprecation', label: type)
-      warn_once(type, msg)
+    # The following methods do the actual warning.
+    #
+    private_class_method def self.do_warn(msg, id)
+      return if @disable_warnings.include?(id)
+      logger(self).warn(msg)
+    end
+
+    private_class_method def self.do_warn_once(msg, id)
+      return unless @warnings.add?(id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate_once(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn_once(msg, id)
     end
   end
 end