bolt 2.42.0 → 2.44.0

data/lib/bolt/config/options.rb

@@ -58,11 +58,32 @@ module Bolt
                        "plan function or the `bolt apply` command.",
           type: Hash,
           properties: {
+            "evaltrace" => {
+              description: "Whether each resource should log when it is being evaluated.",
+              type: [TrueClass, FalseClass],
+              _example: true,
+              _default: false
+            },
+            "log_level" => {
+              description: "The log level for logs in apply reports from Puppet. These can be seen "\
+                           "in ApplyResults.",
+              type: String,
+              enum: %w[debug info notice warning err alert emerg crit],
+              _example: "debug",
+              _default: "notice"
+            },
             "show_diff" => {
               description: "Whether to log and report a contextual diff.",
               type: [TrueClass, FalseClass],
               _example: true,
               _default: false
+            },
+            "trace" => {
+              description: "Whether to print stack traces on some errors. Will print internal Ruby "\
+                           "stack trace interleaved with Puppet function frames.",
+              type: [TrueClass, FalseClass],
+              _example: true,
+              _default: false
             }
           },
           _plugin: false,
@@ -73,11 +94,33 @@ module Bolt
                        "plan function or the `bolt apply` command.",
           type: Hash,
           properties: {
+            "evaltrace" => {
+              description: "Whether each resource should log when it is being evaluated. This allows "\
+                           "you to interactively see exactly what is being done.",
+              type: [TrueClass, FalseClass],
+              _example: true,
+              _default: false
+            },
+            "log_level" => {
+              description: "The log level for logs in apply reports from Puppet. These can be seen "\
+                           "in ApplyResults.",
+              type: String,
+              enum: %w[debug info notice warning err alert emerg crit],
+              _example: "debug",
+              _default: "notice"
+            },
             "show_diff" => {
               description: "Whether to log and report a contextual diff.",
               type: [TrueClass, FalseClass],
               _example: true,
               _default: false
+            },
+            "trace" => {
+              description: "Whether to print stack traces on some errors. Will print internal Ruby "\
+                           "stack trace interleaved with Puppet function frames.",
+              type: [TrueClass, FalseClass],
+              _example: true,
+              _default: false
             }
           },
           _plugin: false
@@ -105,6 +148,17 @@ module Bolt
           _example: 50,
           _default: "100 or 1/7 the ulimit, whichever is lower."
         },
+        "disable-warnings" => {
+          description: "An array of IDs of warnings to suppress. Warnings with a matching ID will not be logged "\
+                       "by Bolt. If you are upgrading Bolt to a new major version, you should re-enable all warnings "\
+                       "until you have finished upgrading.",
+          type: Array,
+          items: {
+            type: String
+          },
+          _plugin: false,
+          _example: ["powershell_2"]
+        },
         "format" => {
           description: "The format to use when printing results.",
           type: String,
@@ -260,6 +314,10 @@ module Bolt
                     description: "The name of the module.",
                     type: String
                   },
+                  "resolve" => {
+                    description: "Whether to resolve the module's dependencies when installing modules.",
+                    type: [TrueClass, FalseClass]
+                  },
                   "version_requirement" => {
                     description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
                                  "shorthand (1.2.x), or a version range (>= 1.2.0).",
@@ -274,9 +332,17 @@ module Bolt
                     description: "The URL to the public git repository.",
                     type: String
                   },
+                  "name" => {
+                    description: "The name of the module. Required when `resolve` is `false`.",
+                    type: String
+                  },
                   "ref" => {
                     description: "The git reference to check out. Can be either a branch, tag, or commit SHA.",
                     type: String
+                  },
+                  "resolve" => {
+                    description: "Whether to resolve the module's dependencies when installing modules.",
+                    type: [TrueClass, FalseClass]
                   }
                 }
               }
@@ -556,6 +622,7 @@ module Bolt
         color
         compile-concurrency
         concurrency
+        disable-warnings
         format
         inventory-config
         log
@@ -577,6 +644,7 @@ module Bolt
         color
         compile-concurrency
         concurrency
+        disable-warnings
         format
         hiera-config
         inventoryfile

data/lib/bolt/config/transport/options.rb

@@ -253,6 +253,13 @@ module Bolt
             _plugin: true,
             _example: "jump.example.com"
           },
+          "read-timeout" => {
+            type: Integer,
+            description: "How long to wait in seconds when making requests to the Orchestrator.",
+            minimum: 1,
+            _plugin: true,
+            _example: 15
+          },
           "realm" => {
             type: String,
             description: "The Kerberos realm (Active Directory domain) to authenticate against.",

data/lib/bolt/config/transport/orch.rb

@@ -12,6 +12,7 @@ module Bolt
           host
           job-poll-interval
           job-poll-timeout
+          read-timeout
           service-url
           task-environment
           token-file

data/lib/bolt/executor.rb

@@ -100,6 +100,8 @@ module Bolt
         # that type of event, publish the event
         next unless types.nil? || types.include?(event[:type])
         @publisher.post(subscriber) do |sub|
+          # Wait for user to input to prompt before printing anything
+          sleep(0.1) while @prompting
           sub.handle_event(event)
         end
       end
@@ -119,11 +121,12 @@ module Bolt
     def queue_execute(targets)
       if @warn_concurrency && targets.length > @concurrency
         @warn_concurrency = false
-        @logger.warn("The ulimit is low, which may cause file limit issues. Default concurrency has been set to "\
-                     "'#{@concurrency}' to mitigate those issues, which may cause Bolt to run slow. "\
-                     "Disable this warning by configuring ulimit using 'ulimit -n <limit>' in your shell "\
-                     "configuration, or by configuring Bolt's concurrency. "\
-                     "See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details.")
+        msg = "The ulimit is low, which may cause file limit issues. Default concurrency has been set to "\
+              "'#{@concurrency}' to mitigate those issues, which may cause Bolt to run slow. "\
+              "Disable this warning by configuring ulimit using 'ulimit -n <limit>' in your shell "\
+              "configuration, or by configuring Bolt's concurrency. "\
+              "See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+        Bolt::Logger.warn("low_ulimit", msg)
       end
 
       targets.group_by(&:transport).flat_map do |protocol, protocol_targets|
@@ -258,7 +261,9 @@ module Bolt
 
     def with_node_logging(description, batch, log_level = :info)
       @logger.send(log_level, "#{description} on #{batch.map(&:safe_name)}")
+      publish_event(type: :start_spin)
       result = yield
+      publish_event(type: :stop_spin)
       @logger.send(log_level, result.to_json)
       result
     end
@@ -410,6 +415,7 @@ module Bolt
       subscribe(self, [:node_result])
       results = Array.new(skein.length)
       @in_parallel = true
+      publish_event(type: :stop_spin)
 
       until skein.empty?
         @thread_completed = false
@@ -417,6 +423,7 @@ module Bolt
 
         skein.each do |yarn|
           if yarn.alive?
+            publish_event(type: :stop_spin)
             r = yarn.resume
           else
             results[yarn.index] = yarn.value
@@ -428,6 +435,7 @@ module Bolt
         sleep(0.1) until @thread_completed || skein.empty?
       end
 
+      publish_event(type: :stop_spin)
       @in_parallel = false
       unsubscribe(self, [:node_result])
       results
@@ -469,6 +477,7 @@ module Bolt
     end
 
     def prompt(prompt, options)
+      @prompting = true
       unless $stdin.tty?
         raise Bolt::Error.new('STDIN is not a tty, unable to prompt', 'bolt/no-tty-error')
       end
@@ -480,6 +489,7 @@ module Bolt
               else
                 $stdin.gets.to_s.chomp
               end
+      @prompting = false
 
       $stderr.puts if options[:sensitive]
 

data/lib/bolt/inventory.rb

@@ -97,7 +97,7 @@ module Bolt
 
       Bolt::Validator.new.tap do |validator|
         validator.validate(data, schema, source)
-        validator.warnings.each { |warning| logger.warn(warning) }
+        validator.warnings.each { |warning| Bolt::Logger.warn(warning[:id], warning[:msg]) }
       end
 
       inventory = create_version(data, config.transport, config.transports, plugins)

data/lib/bolt/inventory/group.rb

@@ -27,7 +27,10 @@ module Bolt
 
         if all_group
           if input.key?('name') && input['name'] != 'all'
-            @logger.warn("Top-level group '#{input['name']}' cannot specify a name, using 'all' instead.")
+            Bolt::Logger.warn(
+              "top_level_group_name",
+              "Top-level group '#{input['name']}' cannot specify a name, using 'all' instead."
+            )
           end
 
           input = input.merge('name' => 'all')
@@ -134,7 +137,7 @@ module Bolt
 
         unless (unexpected_keys = target.keys - TARGET_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in target #{t_name}"
-          @logger.warn(msg)
+          Bolt::Logger.warn("unknown_target_keys", msg)
         end
 
         validate_data_keys(target, t_name)
@@ -261,7 +264,7 @@ module Bolt
 
         unless (unexpected_keys = input.keys - GROUP_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in group #{@name}"
-          @logger.warn(msg)
+          Bolt::Logger.warn("unknown_group_keys", msg)
         end
       end
 
@@ -368,7 +371,7 @@ module Bolt
             msg = +"Found unexpected key(s) #{unexpected_keys.join(', ')} in config for"
             msg << " target #{target} in" if target
             msg << " group #{@name}"
-            @logger.warn(msg)
+            Bolt::Logger.warn("unknown_config_keys", msg)
           end
         end
       end

data/lib/bolt/logger.rb

@@ -5,8 +5,14 @@ require 'logging'
 module Bolt
   module Logger
     LEVELS = %w[trace debug info notice warn error fatal].freeze
-    @mutex = Mutex.new
-    @warnings = Set.new
+
+    # This module is treated as a global singleton so that multiple classes
+    # in Bolt can log warnings with IDs. Access to the following variables
+    # are controlled by a mutex.
+    @mutex            = Mutex.new
+    @warnings         = Set.new
+    @disable_warnings = Set.new
+    @message_queue    = []
 
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
@@ -36,7 +42,7 @@ module Bolt
       end
     end
 
-    def self.configure(destinations, color)
+    def self.configure(destinations, color, disable_warnings = nil)
       root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
@@ -73,6 +79,16 @@ module Bolt
 
         appender.level = params[:level] if params[:level]
       end
+
+      # Set the list of disabled warnings and mark the logger as configured.
+      # Log all messages in the message queue and flush the queue.
+      if disable_warnings
+        @mutex.synchronize { @disable_warnings = disable_warnings }
+      end
+    end
+
+    def self.configured?
+      Logging.logger[:root].appenders.any?
     end
 
     # A helper to ensure the Logging library is always initialized with our
@@ -123,18 +139,106 @@ module Bolt
       Logging.reset
     end
 
-    def self.warn_once(type, msg)
+    # The following methods are used in place of the Logging.logger
+    # methods of the same name when logging warning messages or logging
+    # any messages prior to the logger being configured. If the logger
+    # is not configured when any of these methods are called, the message
+    # will be added to a queue, otherwise they are logged immediately.
+    # The message queue is flushed by calling #flush_queue, which is
+    # called from Bolt::CLI after configuring the logger.
+    #
+    def self.warn(id, msg)
+      log(type: :warn, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.warn_once(id, msg)
+      log(type: :warn_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate(id, msg)
+      log(type: :deprecate, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate_once(id, msg)
+      log(type: :deprecate_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.debug(msg)
+      log(type: :debug, msg: msg)
+    end
+
+    def self.info(msg)
+      log(type: :info, msg: msg)
+    end
+
+    # Logs a message. If the logger has not been configured, this will queue
+    # the message to be logged later. Once the logger is configured, the
+    # queue will be flushed of all messages and new messages will be logged
+    # immediately.
+    #
+    # Logging with this method is controlled by a mutex, as the Bolt::Logger
+    # module is treated as a global singleton to allow multiple classes
+    # access to its methods.
+    #
+    private_class_method def self.log(type:, msg:, id: nil)
+      @mutex.synchronize do
+        if configured?
+          log_message(type: type, msg: msg, id: id)
+        else
+          @message_queue << { type: type, msg: msg, id: id }
+        end
+      end
+    end
+
+    # Logs all messages in the message queue and then flushes the queue.
+    #
+    def self.flush_queue
       @mutex.synchronize do
-        @logger ||= Bolt::Logger.logger(self)
-        if @warnings.add?(type)
-          @logger.warn(msg)
+        @message_queue.each do |message|
+          log_message(message)
         end
+
+        @message_queue.clear
+      end
+    end
+
+    # Handles the actual logging of a message.
+    #
+    private_class_method def self.log_message(type:, msg:, id: nil)
+      case type
+      when :warn
+        do_warn(msg, id)
+      when :warn_once
+        do_warn_once(msg, id)
+      when :deprecate
+        do_deprecate(msg, id)
+      when :deprecate_once
+        do_deprecate_once(msg, id)
+      else
+        logger(self).send(type, msg)
       end
     end
 
-    def self.deprecation_warning(type, msg)
-      @analytics&.event('Warn', 'deprecation', label: type)
-      warn_once(type, msg)
+    # The following methods do the actual warning.
+    #
+    private_class_method def self.do_warn(msg, id)
+      return if @disable_warnings.include?(id)
+      logger(self).warn(msg)
+    end
+
+    private_class_method def self.do_warn_once(msg, id)
+      return unless @warnings.add?(id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate_once(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn_once(msg, id)
     end
   end
 end

data/lib/bolt/module_installer.rb

@@ -195,8 +195,10 @@ module Bolt
       @outputter.stop_spin
 
       # Automatically generate types after installing modules
-      @outputter.print_action_step("Generating type references")
-      @pal.generate_types
+      if ok
+        @outputter.print_action_step("Generating type references")
+        @pal.generate_types(cache: true)
+      end
 
       @outputter.print_puppetfile_result(ok, path, moduledir)
 

data/lib/bolt/module_installer/resolver.rb

@@ -13,10 +13,15 @@ module Bolt
         require 'puppetfile-resolver'
 
         # Build the document model from the specs.
-        document = PuppetfileResolver::Puppetfile::Document.new('')
+        document   = PuppetfileResolver::Puppetfile::Document.new('')
+        unresolved = []
 
         specs.specs.each do |spec|
-          document.add_module(spec.to_resolver_module)
+          if spec.resolve
+            document.add_module(spec.to_resolver_module)
+          else
+            unresolved << spec
+          end
         end
 
         # Make sure the document model is valid.
@@ -47,20 +52,40 @@ module Bolt
           raise Bolt::Error.new(e.message, 'bolt/module-resolver-error')
         end
 
-        # Convert the specs returned from the resolver into Bolt module objects.
-        modules = result.specifications.values.each_with_object([]) do |mod, acc|
+        # Create the Puppetfile object.
+        generate_puppetfile(specs, result.specifications.values, unresolved)
+      end
+
+      # Creates a puppetfile-resolver config object.
+      #
+      private def spec_searcher_config(config)
+        PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
+          obj.forge.proxy     = config.dig('forge', 'proxy') || config.dig('proxy')
+          obj.git.proxy       = config.dig('proxy')
+          obj.forge.forge_api = config.dig('forge', 'baseurl')
+        end
+      end
+
+      # Creates a Puppetfile object with Module objects created from resolved and
+      # unresolved specs.
+      #
+      private def generate_puppetfile(specs, resolved, unresolved)
+        modules = []
+
+        # Convert the resolved specs into Bolt module objects.
+        resolved.each do |mod|
           # Skip over anything that isn't a module spec, such as a Puppet spec.
           next unless mod.is_a? PuppetfileResolver::Models::ModuleSpecification
 
           case mod.origin
           when :forge
-            acc << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
+            modules << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
               "#{mod.owner}/#{mod.name}",
               mod.version.to_s
             )
           when :git
             spec = specs.specs.find { |s| s.name == mod.name }
-            acc << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
+            modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
               spec.name,
               spec.git,
               spec.sha
@@ -68,16 +93,36 @@ module Bolt
           end
         end
 
-        # Create the Puppetfile object.
-        Bolt::ModuleInstaller::Puppetfile.new(modules)
-      end
+        # Error if there are any name conflicts between unresolved specs and
+        # resolved modules. r10k will error if a Puppetfile includes duplicate
+        # names, but we error early here to provide a more helpful message.
+        if (name_conflicts = modules.map(&:name) & unresolved.map(&:name)).any?
+          raise Bolt::Error.new(
+            "Detected unresolved module specifications with the same name as a resolved module "\
+            "dependency: #{name_conflicts.join(', ')}. Either remove the unresolved module specification "\
+            "or set the module with the conflicting dependency to not resolve.",
+            "bolt/module-name-conflict-error"
+          )
+        end
 
-      private def spec_searcher_config(config)
-        PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
-          obj.forge.proxy     = config.dig('forge', 'proxy') || config.dig('proxy')
-          obj.git.proxy       = config.dig('proxy')
-          obj.forge.forge_api = config.dig('forge', 'baseurl')
+        # Convert the unresolved specs into Bolt module objects.
+        unresolved.each do |spec|
+          case spec.type
+          when :forge
+            modules << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
+              spec.full_name,
+              spec.version_requirement
+            )
+          when :git
+            modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
+              spec.name,
+              spec.git,
+              spec.ref
+            )
+          end
         end
+
+        Bolt::ModuleInstaller::Puppetfile.new(modules)
       end
     end
   end