bolt 2.42.0 → 3.3.0

data/lib/bolt/inventory.rb

@@ -97,7 +97,7 @@ module Bolt
 
       Bolt::Validator.new.tap do |validator|
         validator.validate(data, schema, source)
-        validator.warnings.each { |warning| logger.warn(warning) }
+        validator.warnings.each { |warning| Bolt::Logger.warn(warning[:id], warning[:msg]) }
       end
 
       inventory = create_version(data, config.transport, config.transports, plugins)

data/lib/bolt/inventory/group.rb

@@ -27,7 +27,10 @@ module Bolt
 
         if all_group
           if input.key?('name') && input['name'] != 'all'
-            @logger.warn("Top-level group '#{input['name']}' cannot specify a name, using 'all' instead.")
+            Bolt::Logger.warn(
+              "top_level_group_name",
+              "Top-level group '#{input['name']}' cannot specify a name, using 'all' instead."
+            )
           end
 
           input = input.merge('name' => 'all')
@@ -134,7 +137,7 @@ module Bolt
 
         unless (unexpected_keys = target.keys - TARGET_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in target #{t_name}"
-          @logger.warn(msg)
+          Bolt::Logger.warn("unknown_target_keys", msg)
         end
 
         validate_data_keys(target, t_name)
@@ -261,7 +264,7 @@ module Bolt
 
         unless (unexpected_keys = input.keys - GROUP_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in group #{@name}"
-          @logger.warn(msg)
+          Bolt::Logger.warn("unknown_group_keys", msg)
         end
       end
 
@@ -368,7 +371,7 @@ module Bolt
             msg = +"Found unexpected key(s) #{unexpected_keys.join(', ')} in config for"
             msg << " target #{target} in" if target
             msg << " group #{@name}"
-            @logger.warn(msg)
+            Bolt::Logger.warn("unknown_config_keys", msg)
           end
         end
       end

data/lib/bolt/logger.rb

@@ -4,9 +4,15 @@ require 'logging'
 
 module Bolt
   module Logger
-    LEVELS = %w[trace debug info notice warn error fatal].freeze
-    @mutex = Mutex.new
-    @warnings = Set.new
+    LEVELS = %w[trace debug info warn error fatal].freeze
+
+    # This module is treated as a global singleton so that multiple classes
+    # in Bolt can log warnings with IDs. Access to the following variables
+    # are controlled by a mutex.
+    @mutex            = Mutex.new
+    @warnings         = Set.new
+    @disable_warnings = Set.new
+    @message_queue    = []
 
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
@@ -36,7 +42,7 @@ module Bolt
       end
     end
 
-    def self.configure(destinations, color)
+    def self.configure(destinations, color, disable_warnings = nil)
       root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
@@ -73,6 +79,24 @@ module Bolt
 
         appender.level = params[:level] if params[:level]
       end
+
+      # Set the list of disabled warnings and mark the logger as configured.
+      # Log all messages in the message queue and flush the queue.
+      if disable_warnings
+        @mutex.synchronize { @disable_warnings = disable_warnings }
+      end
+    end
+
+    def self.configured?
+      Logging.logger[:root].appenders.any?
+    end
+
+    def self.stream
+      @stream
+    end
+
+    def self.stream=(stream)
+      @stream = stream
     end
 
     # A helper to ensure the Logging library is always initialized with our
@@ -123,18 +147,106 @@ module Bolt
       Logging.reset
     end
 
-    def self.warn_once(type, msg)
+    # The following methods are used in place of the Logging.logger
+    # methods of the same name when logging warning messages or logging
+    # any messages prior to the logger being configured. If the logger
+    # is not configured when any of these methods are called, the message
+    # will be added to a queue, otherwise they are logged immediately.
+    # The message queue is flushed by calling #flush_queue, which is
+    # called from Bolt::CLI after configuring the logger.
+    #
+    def self.warn(id, msg)
+      log(type: :warn, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.warn_once(id, msg)
+      log(type: :warn_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate(id, msg)
+      log(type: :deprecate, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate_once(id, msg)
+      log(type: :deprecate_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.debug(msg)
+      log(type: :debug, msg: msg)
+    end
+
+    def self.info(msg)
+      log(type: :info, msg: msg)
+    end
+
+    # Logs a message. If the logger has not been configured, this will queue
+    # the message to be logged later. Once the logger is configured, the
+    # queue will be flushed of all messages and new messages will be logged
+    # immediately.
+    #
+    # Logging with this method is controlled by a mutex, as the Bolt::Logger
+    # module is treated as a global singleton to allow multiple classes
+    # access to its methods.
+    #
+    private_class_method def self.log(type:, msg:, id: nil)
       @mutex.synchronize do
-        @logger ||= Bolt::Logger.logger(self)
-        if @warnings.add?(type)
-          @logger.warn(msg)
+        if configured?
+          log_message(type: type, msg: msg, id: id)
+        else
+          @message_queue << { type: type, msg: msg, id: id }
         end
       end
     end
 
-    def self.deprecation_warning(type, msg)
-      @analytics&.event('Warn', 'deprecation', label: type)
-      warn_once(type, msg)
+    # Logs all messages in the message queue and then flushes the queue.
+    #
+    def self.flush_queue
+      @mutex.synchronize do
+        @message_queue.each do |message|
+          log_message(message)
+        end
+
+        @message_queue.clear
+      end
+    end
+
+    # Handles the actual logging of a message.
+    #
+    private_class_method def self.log_message(type:, msg:, id: nil)
+      case type
+      when :warn
+        do_warn(msg, id)
+      when :warn_once
+        do_warn_once(msg, id)
+      when :deprecate
+        do_deprecate(msg, id)
+      when :deprecate_once
+        do_deprecate_once(msg, id)
+      else
+        logger(self).send(type, msg)
+      end
+    end
+
+    # The following methods do the actual warning.
+    #
+    private_class_method def self.do_warn(msg, id)
+      return if @disable_warnings.include?(id)
+      logger(self).warn(msg)
+    end
+
+    private_class_method def self.do_warn_once(msg, id)
+      return unless @warnings.add?(id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate_once(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn_once(msg, id)
     end
   end
 end

data/lib/bolt/module_installer.rb

@@ -45,7 +45,7 @@ module Bolt
       # specss. If that fails, fall back to resolving from project specs.
       # This prevents Bolt from modifying installed modules unless there is
       # a version conflict.
-      @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+      @outputter.print_action_step("Resolving module dependencies, this might take a moment")
 
       @outputter.start_spin
       begin
@@ -156,7 +156,7 @@ module Bolt
         # If forcibly installing or if there is no Puppetfile, resolve
         # and write a Puppetfile.
         if force || !path.exist?
-          @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+          @outputter.print_action_step("Resolving module dependencies, this might take a moment")
 
           # This doesn't use the block as it's more testable to just mock *_spin
           @outputter.start_spin
@@ -195,8 +195,10 @@ module Bolt
       @outputter.stop_spin
 
       # Automatically generate types after installing modules
-      @outputter.print_action_step("Generating type references")
-      @pal.generate_types
+      if ok
+        @outputter.print_action_step("Generating type references")
+        @pal.generate_types(cache: true)
+      end
 
       @outputter.print_puppetfile_result(ok, path, moduledir)
 

data/lib/bolt/module_installer/puppetfile.rb

@@ -36,7 +36,7 @@ module Bolt
           raise Bolt::ValidationError, <<~MSG
             Unable to parse Puppetfile #{path}:
             #{parsed.validation_errors.join("\n\n")}.
-            This may not be a Puppetfile managed by Bolt.
+            This Puppetfile might not be managed by Bolt.
           MSG
         end
 
@@ -106,7 +106,7 @@ module Bolt
 
           #{unsatisfied_specs.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
           
-          This may not be a Puppetfile managed by Bolt. To forcibly overwrite the
+          This Puppetfile might not be managed by Bolt. To forcibly overwrite the
           Puppetfile, run '#{command}'.
         MESSAGE
 

data/lib/bolt/module_installer/resolver.rb

@@ -13,10 +13,15 @@ module Bolt
         require 'puppetfile-resolver'
 
         # Build the document model from the specs.
-        document = PuppetfileResolver::Puppetfile::Document.new('')
+        document   = PuppetfileResolver::Puppetfile::Document.new('')
+        unresolved = []
 
         specs.specs.each do |spec|
-          document.add_module(spec.to_resolver_module)
+          if spec.resolve
+            document.add_module(spec.to_resolver_module)
+          else
+            unresolved << spec
+          end
         end
 
         # Make sure the document model is valid.
@@ -47,20 +52,40 @@ module Bolt
           raise Bolt::Error.new(e.message, 'bolt/module-resolver-error')
         end
 
-        # Convert the specs returned from the resolver into Bolt module objects.
-        modules = result.specifications.values.each_with_object([]) do |mod, acc|
+        # Create the Puppetfile object.
+        generate_puppetfile(specs, result.specifications.values, unresolved)
+      end
+
+      # Creates a puppetfile-resolver config object.
+      #
+      private def spec_searcher_config(config)
+        PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
+          obj.forge.proxy     = config.dig('forge', 'proxy') || config.dig('proxy')
+          obj.git.proxy       = config.dig('proxy')
+          obj.forge.forge_api = config.dig('forge', 'baseurl')
+        end
+      end
+
+      # Creates a Puppetfile object with Module objects created from resolved and
+      # unresolved specs.
+      #
+      private def generate_puppetfile(specs, resolved, unresolved)
+        modules = []
+
+        # Convert the resolved specs into Bolt module objects.
+        resolved.each do |mod|
           # Skip over anything that isn't a module spec, such as a Puppet spec.
           next unless mod.is_a? PuppetfileResolver::Models::ModuleSpecification
 
           case mod.origin
           when :forge
-            acc << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
+            modules << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
               "#{mod.owner}/#{mod.name}",
               mod.version.to_s
             )
           when :git
             spec = specs.specs.find { |s| s.name == mod.name }
-            acc << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
+            modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
               spec.name,
               spec.git,
               spec.sha
@@ -68,16 +93,36 @@ module Bolt
           end
         end
 
-        # Create the Puppetfile object.
-        Bolt::ModuleInstaller::Puppetfile.new(modules)
-      end
+        # Error if there are any name conflicts between unresolved specs and
+        # resolved modules. r10k will error if a Puppetfile includes duplicate
+        # names, but we error early here to provide a more helpful message.
+        if (name_conflicts = modules.map(&:name) & unresolved.map(&:name)).any?
+          raise Bolt::Error.new(
+            "Detected unresolved module specifications with the same name as a resolved module "\
+            "dependency: #{name_conflicts.join(', ')}. Either remove the unresolved module specification "\
+            "or set the module with the conflicting dependency to not resolve.",
+            "bolt/module-name-conflict-error"
+          )
+        end
 
-      private def spec_searcher_config(config)
-        PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
-          obj.forge.proxy     = config.dig('forge', 'proxy') || config.dig('proxy')
-          obj.git.proxy       = config.dig('proxy')
-          obj.forge.forge_api = config.dig('forge', 'baseurl')
+        # Convert the unresolved specs into Bolt module objects.
+        unresolved.each do |spec|
+          case spec.type
+          when :forge
+            modules << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
+              spec.full_name,
+              spec.version_requirement
+            )
+          when :git
+            modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
+              spec.name,
+              spec.git,
+              spec.ref
+            )
+          end
         end
+
+        Bolt::ModuleInstaller::Puppetfile.new(modules)
       end
     end
   end

data/lib/bolt/module_installer/specs/forge_spec.rb

@@ -13,14 +13,20 @@ module Bolt
       class ForgeSpec
         NAME_REGEX    = %r{\A[a-zA-Z0-9]+[-/](?<name>[a-z][a-z0-9_]*)\z}.freeze
         REQUIRED_KEYS = Set.new(%w[name]).freeze
-        KNOWN_KEYS    = Set.new(%w[name version_requirement]).freeze
+        KNOWN_KEYS    = Set.new(%w[name resolve version_requirement]).freeze
 
-        attr_reader :full_name, :name, :semantic_version, :type
+        attr_reader :full_name, :name, :resolve, :semantic_version, :type, :version_requirement
 
         def initialize(init_hash)
+          @resolve                                = init_hash.key?('resolve') ? init_hash['resolve'] : true
           @full_name, @name                       = parse_name(init_hash['name'])
           @version_requirement, @semantic_version = parse_version_requirement(init_hash['version_requirement'])
           @type                                   = :forge
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@full_name} must be a Boolean"
+          end
         end
 
         def self.implements?(hash)
@@ -33,8 +39,8 @@ module Bolt
           unless (match = name.match(NAME_REGEX))
             raise Bolt::ValidationError,
                   "Invalid name for Forge module specification: #{name}. Name must match "\
-                  "'owner/name'. Owner segment may only include letters or digits. Name "\
-                  "segment must start with a lowercase letter and may only include lowercase "\
+                  "'owner/name'. Owner segment can only include letters or digits. Name "\
+                  "segment must start with a lowercase letter and can only include lowercase "\
                   "letters, digits, and underscores."
           end
 

data/lib/bolt/module_installer/specs/git_spec.rb

@@ -13,18 +13,31 @@ module Bolt
       class GitSpec
         NAME_REGEX    = %r{\A(?:[a-zA-Z0-9]+[-/])?(?<name>[a-z][a-z0-9_]*)\z}.freeze
         REQUIRED_KEYS = Set.new(%w[git ref]).freeze
+        KNOWN_KEYS    = Set.new(%w[git name ref resolve]).freeze
 
-        attr_reader :git, :ref, :type
+        attr_reader :git, :ref, :resolve, :type
 
         def initialize(init_hash)
+          @resolve    = init_hash.key?('resolve') ? init_hash['resolve'] : true
           @name       = parse_name(init_hash['name'])
           @git, @repo = parse_git(init_hash['git'])
           @ref        = init_hash['ref']
           @type       = :git
+
+          if @name.nil? && @resolve == false
+            raise Bolt::ValidationError,
+                  "Missing name for Git module specification: #{@git}. Git module specifications "\
+                  "must include a 'name' key when 'resolve' is false."
+          end
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+          end
         end
 
         def self.implements?(hash)
-          REQUIRED_KEYS == hash.keys.to_set
+          KNOWN_KEYS.superset?(hash.keys.to_set) && REQUIRED_KEYS.subset?(hash.keys.to_set)
         end
 
         # Parses the name into owner and name segments, and formats the full
@@ -36,8 +49,8 @@ module Bolt
           unless (match = name.match(NAME_REGEX))
             raise Bolt::ValidationError,
                   "Invalid name for Git module specification: #{name}. Name must match "\
-                  "'name' or 'owner/name'. Owner segment may only include letters or digits. "\
-                  "Name segment must start with a lowercase letter and may only include "\
+                  "'name' or 'owner/name'. Owner segment can only include letters or digits. "\
+                  "Name segment must start with a lowercase letter and can only include "\
                   "lowercase letters, digits, and underscores."
           end
 
@@ -47,6 +60,8 @@ module Bolt
         # Gets the repo from the git URL.
         #
         private def parse_git(git)
+          return [git, nil] unless @resolve
+
           repo = if git.start_with?('git@github.com:')
                    git.split('git@github.com:').last.split('.git').first
                  elsif git.start_with?('https://github.com')