bolt 2.38.0 → 3.0.0

data/lib/bolt/inventory/group.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'bolt/config/options'
 require 'bolt/inventory/group'
 require 'bolt/inventory/inventory'
 require 'bolt/inventory/target'
@@ -18,12 +19,23 @@ module Bolt
       GROUP_KEYS = DATA_KEYS + %w[name groups targets]
       CONFIG_KEYS = Bolt::Config::INVENTORY_OPTIONS.keys
 
-      def initialize(input, plugins)
+      def initialize(input, plugins, all_group: false)
         @logger = Bolt::Logger.logger(self)
         @plugins = plugins
 
         input = @plugins.resolve_top_level_references(input) if @plugins.reference?(input)
 
+        if all_group
+          if input.key?('name') && input['name'] != 'all'
+            Bolt::Logger.warn(
+              "top_level_group_name",
+              "Top-level group '#{input['name']}' cannot specify a name, using 'all' instead."
+            )
+          end
+
+          input = input.merge('name' => 'all')
+        end
+
         raise ValidationError.new("Group does not have a name", nil) unless input.key?('name')
 
         @name = @plugins.resolve_references(input['name'])
@@ -125,7 +137,7 @@ module Bolt
 
         unless (unexpected_keys = target.keys - TARGET_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in target #{t_name}"
-          @logger.warn(msg)
+          Bolt::Logger.warn("unknown_target_keys", msg)
         end
 
         validate_data_keys(target, t_name)
@@ -252,15 +264,7 @@ module Bolt
 
         unless (unexpected_keys = input.keys - GROUP_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in group #{@name}"
-          @logger.warn(msg)
-        end
-
-        Bolt::Util.walk_keys(input) do |key|
-          if @plugins.reference?(key)
-            raise ValidationError.new("Group keys cannot be specified as _plugin references", @name)
-          else
-            key
-          end
+          Bolt::Logger.warn("unknown_group_keys", msg)
         end
       end
 
@@ -323,7 +327,26 @@ module Bolt
           'features' => @plugins.resolve_references(data.fetch('features', [])),
           'plugin_hooks' => @plugins.resolve_references(data.fetch('plugin_hooks', {}))
         }
+
         validate_data_keys(result, target)
+
+        Bolt::Config::Options::TRANSPORT_CONFIG.each_key do |transport|
+          next unless result['config'].key?(transport)
+          transport_config = result['config'][transport]
+          next unless transport_config.is_a?(Hash)
+          transport_config = Bolt::Util.postwalk_vals(transport_config) do |val|
+            if val.is_a?(Hash)
+              val = val.compact
+              val = nil if val.empty?
+            end
+            val
+          end
+          # the transport config is user-specified data so we
+          # still want to preserve it even if it exclusively
+          # contains nil-resolved keys
+          result['config'][transport] = transport_config || {}
+        end
+
         result['features'] = Set.new(result['features'].flatten)
         result
       end
@@ -348,7 +371,7 @@ module Bolt
             msg = +"Found unexpected key(s) #{unexpected_keys.join(', ')} in config for"
             msg << " target #{target} in" if target
             msg << " group #{@name}"
-            @logger.warn(msg)
+            Bolt::Logger.warn("unknown_config_keys", msg)
           end
         end
       end

data/lib/bolt/inventory/inventory.rb

@@ -21,7 +21,7 @@ module Bolt
         @transport    = transport
         @config       = transports
         @plugins      = plugins
-        @groups       = Group.new(@data.merge('name' => 'all'), plugins)
+        @groups       = Group.new(@data, plugins, all_group: true)
         @group_lookup = {}
         @targets      = {}
 

data/lib/bolt/logger.rb

@@ -4,9 +4,15 @@ require 'logging'
 
 module Bolt
   module Logger
-    LEVELS = %w[trace debug info notice warn error fatal].freeze
-    @mutex = Mutex.new
-    @warnings = Set.new
+    LEVELS = %w[trace debug info warn error fatal].freeze
+
+    # This module is treated as a global singleton so that multiple classes
+    # in Bolt can log warnings with IDs. Access to the following variables
+    # are controlled by a mutex.
+    @mutex            = Mutex.new
+    @warnings         = Set.new
+    @disable_warnings = Set.new
+    @message_queue    = []
 
     # This method provides a single point-of-entry to setup logging for both
     # the CLI and for tests. This is necessary because we define custom log
@@ -36,7 +42,7 @@ module Bolt
       end
     end
 
-    def self.configure(destinations, color)
+    def self.configure(destinations, color, disable_warnings = nil)
       root_logger = Bolt::Logger.logger(:root)
 
       root_logger.add_appenders Logging.appenders.stderr(
@@ -73,6 +79,16 @@ module Bolt
 
         appender.level = params[:level] if params[:level]
       end
+
+      # Set the list of disabled warnings and mark the logger as configured.
+      # Log all messages in the message queue and flush the queue.
+      if disable_warnings
+        @mutex.synchronize { @disable_warnings = disable_warnings }
+      end
+    end
+
+    def self.configured?
+      Logging.logger[:root].appenders.any?
     end
 
     # A helper to ensure the Logging library is always initialized with our
@@ -123,18 +139,106 @@ module Bolt
       Logging.reset
     end
 
-    def self.warn_once(type, msg)
+    # The following methods are used in place of the Logging.logger
+    # methods of the same name when logging warning messages or logging
+    # any messages prior to the logger being configured. If the logger
+    # is not configured when any of these methods are called, the message
+    # will be added to a queue, otherwise they are logged immediately.
+    # The message queue is flushed by calling #flush_queue, which is
+    # called from Bolt::CLI after configuring the logger.
+    #
+    def self.warn(id, msg)
+      log(type: :warn, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.warn_once(id, msg)
+      log(type: :warn_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate(id, msg)
+      log(type: :deprecate, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.deprecate_once(id, msg)
+      log(type: :deprecate_once, msg: "#{msg} [ID: #{id}]", id: id)
+    end
+
+    def self.debug(msg)
+      log(type: :debug, msg: msg)
+    end
+
+    def self.info(msg)
+      log(type: :info, msg: msg)
+    end
+
+    # Logs a message. If the logger has not been configured, this will queue
+    # the message to be logged later. Once the logger is configured, the
+    # queue will be flushed of all messages and new messages will be logged
+    # immediately.
+    #
+    # Logging with this method is controlled by a mutex, as the Bolt::Logger
+    # module is treated as a global singleton to allow multiple classes
+    # access to its methods.
+    #
+    private_class_method def self.log(type:, msg:, id: nil)
+      @mutex.synchronize do
+        if configured?
+          log_message(type: type, msg: msg, id: id)
+        else
+          @message_queue << { type: type, msg: msg, id: id }
+        end
+      end
+    end
+
+    # Logs all messages in the message queue and then flushes the queue.
+    #
+    def self.flush_queue
       @mutex.synchronize do
-        @logger ||= Bolt::Logger.logger(self)
-        if @warnings.add?(type)
-          @logger.warn(msg)
+        @message_queue.each do |message|
+          log_message(message)
         end
+
+        @message_queue.clear
+      end
+    end
+
+    # Handles the actual logging of a message.
+    #
+    private_class_method def self.log_message(type:, msg:, id: nil)
+      case type
+      when :warn
+        do_warn(msg, id)
+      when :warn_once
+        do_warn_once(msg, id)
+      when :deprecate
+        do_deprecate(msg, id)
+      when :deprecate_once
+        do_deprecate_once(msg, id)
+      else
+        logger(self).send(type, msg)
       end
     end
 
-    def self.deprecation_warning(type, msg)
-      @analytics&.event('Warn', 'deprecation', label: type)
-      warn_once(type, msg)
+    # The following methods do the actual warning.
+    #
+    private_class_method def self.do_warn(msg, id)
+      return if @disable_warnings.include?(id)
+      logger(self).warn(msg)
+    end
+
+    private_class_method def self.do_warn_once(msg, id)
+      return unless @warnings.add?(id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn(msg, id)
+    end
+
+    private_class_method def self.do_deprecate_once(msg, id)
+      @analytics&.event('Warn', 'deprecation', label: id)
+      do_warn_once(msg, id)
     end
   end
 end

data/lib/bolt/module.rb

@@ -6,8 +6,14 @@ module Bolt
     CONTENT_NAME_REGEX = /\A[a-z][a-z0-9_]*(::[a-z][a-z0-9_]*)*\z/.freeze
     MODULE_NAME_REGEX  = /\A[a-z][a-z0-9_]*\z/.freeze
 
-    def self.discover(modulepath)
-      modulepath.each_with_object({}) do |path, mods|
+    def self.discover(modulepath, project)
+      mods = {}
+
+      if project.load_as_module?
+        mods[project.name] = Bolt::Module.new(project.name, project.path.to_s)
+      end
+
+      modulepath.each do |path|
         next unless File.exist?(path) && File.directory?(path)
         Dir.children(path)
            .map { |dir| File.join(path, dir) }
@@ -20,6 +26,8 @@ module Bolt
           end
         end
       end
+
+      mods
     end
 
     attr_reader :name, :path

data/lib/bolt/module_installer.rb

@@ -195,8 +195,10 @@ module Bolt
       @outputter.stop_spin
 
       # Automatically generate types after installing modules
-      @outputter.print_action_step("Generating type references")
-      @pal.generate_types
+      if ok
+        @outputter.print_action_step("Generating type references")
+        @pal.generate_types(cache: true)
+      end
 
       @outputter.print_puppetfile_result(ok, path, moduledir)
 

data/lib/bolt/module_installer/resolver.rb

@@ -9,14 +9,19 @@ module Bolt
     class Resolver
       # Resolves module specs and returns a Puppetfile object.
       #
-      def resolve(specs, _config = {})
+      def resolve(specs, config = {})
         require 'puppetfile-resolver'
 
         # Build the document model from the specs.
-        document = PuppetfileResolver::Puppetfile::Document.new('')
+        document   = PuppetfileResolver::Puppetfile::Document.new('')
+        unresolved = []
 
         specs.specs.each do |spec|
-          document.add_module(spec.to_resolver_module)
+          if spec.resolve
+            document.add_module(spec.to_resolver_module)
+          else
+            unresolved << spec
+          end
         end
 
         # Make sure the document model is valid.
@@ -38,29 +43,49 @@ module Bolt
         # raised by puppetfile-resolver and re-raising them as Bolt errors.
         begin
           result = resolver.resolve(
-            cache:                 nil,
-            ui:                    nil,
-            module_paths:          [],
-            allow_missing_modules: false
+            cache:                       nil,
+            ui:                          nil,
+            allow_missing_modules:       false,
+            spec_searcher_configuration: spec_searcher_config(config)
           )
         rescue StandardError => e
           raise Bolt::Error.new(e.message, 'bolt/module-resolver-error')
         end
 
-        # Convert the specs returned from the resolver into Bolt module objects.
-        modules = result.specifications.values.each_with_object([]) do |mod, acc|
+        # Create the Puppetfile object.
+        generate_puppetfile(specs, result.specifications.values, unresolved)
+      end
+
+      # Creates a puppetfile-resolver config object.
+      #
+      private def spec_searcher_config(config)
+        PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
+          obj.forge.proxy     = config.dig('forge', 'proxy') || config.dig('proxy')
+          obj.git.proxy       = config.dig('proxy')
+          obj.forge.forge_api = config.dig('forge', 'baseurl')
+        end
+      end
+
+      # Creates a Puppetfile object with Module objects created from resolved and
+      # unresolved specs.
+      #
+      private def generate_puppetfile(specs, resolved, unresolved)
+        modules = []
+
+        # Convert the resolved specs into Bolt module objects.
+        resolved.each do |mod|
           # Skip over anything that isn't a module spec, such as a Puppet spec.
           next unless mod.is_a? PuppetfileResolver::Models::ModuleSpecification
 
           case mod.origin
           when :forge
-            acc << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
+            modules << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
               "#{mod.owner}/#{mod.name}",
               mod.version.to_s
             )
           when :git
             spec = specs.specs.find { |s| s.name == mod.name }
-            acc << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
+            modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
               spec.name,
               spec.git,
               spec.sha
@@ -68,7 +93,35 @@ module Bolt
           end
         end
 
-        # Create the Puppetfile object.
+        # Error if there are any name conflicts between unresolved specs and
+        # resolved modules. r10k will error if a Puppetfile includes duplicate
+        # names, but we error early here to provide a more helpful message.
+        if (name_conflicts = modules.map(&:name) & unresolved.map(&:name)).any?
+          raise Bolt::Error.new(
+            "Detected unresolved module specifications with the same name as a resolved module "\
+            "dependency: #{name_conflicts.join(', ')}. Either remove the unresolved module specification "\
+            "or set the module with the conflicting dependency to not resolve.",
+            "bolt/module-name-conflict-error"
+          )
+        end
+
+        # Convert the unresolved specs into Bolt module objects.
+        unresolved.each do |spec|
+          case spec.type
+          when :forge
+            modules << Bolt::ModuleInstaller::Puppetfile::ForgeModule.new(
+              spec.full_name,
+              spec.version_requirement
+            )
+          when :git
+            modules << Bolt::ModuleInstaller::Puppetfile::GitModule.new(
+              spec.name,
+              spec.git,
+              spec.ref
+            )
+          end
+        end
+
         Bolt::ModuleInstaller::Puppetfile.new(modules)
       end
     end

data/lib/bolt/module_installer/specs/forge_spec.rb

@@ -13,14 +13,20 @@ module Bolt
       class ForgeSpec
         NAME_REGEX    = %r{\A[a-zA-Z0-9]+[-/](?<name>[a-z][a-z0-9_]*)\z}.freeze
         REQUIRED_KEYS = Set.new(%w[name]).freeze
-        KNOWN_KEYS    = Set.new(%w[name version_requirement]).freeze
+        KNOWN_KEYS    = Set.new(%w[name resolve version_requirement]).freeze
 
-        attr_reader :full_name, :name, :semantic_version, :type
+        attr_reader :full_name, :name, :resolve, :semantic_version, :type, :version_requirement
 
         def initialize(init_hash)
+          @resolve                                = init_hash.key?('resolve') ? init_hash['resolve'] : true
           @full_name, @name                       = parse_name(init_hash['name'])
           @version_requirement, @semantic_version = parse_version_requirement(init_hash['version_requirement'])
           @type                                   = :forge
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@full_name} must be a Boolean"
+          end
         end
 
         def self.implements?(hash)

data/lib/bolt/module_installer/specs/git_spec.rb

@@ -13,18 +13,31 @@ module Bolt
       class GitSpec
         NAME_REGEX    = %r{\A(?:[a-zA-Z0-9]+[-/])?(?<name>[a-z][a-z0-9_]*)\z}.freeze
         REQUIRED_KEYS = Set.new(%w[git ref]).freeze
+        KNOWN_KEYS    = Set.new(%w[git name ref resolve]).freeze
 
-        attr_reader :git, :ref, :type
+        attr_reader :git, :ref, :resolve, :type
 
         def initialize(init_hash)
+          @resolve    = init_hash.key?('resolve') ? init_hash['resolve'] : true
           @name       = parse_name(init_hash['name'])
           @git, @repo = parse_git(init_hash['git'])
           @ref        = init_hash['ref']
           @type       = :git
+
+          if @name.nil? && @resolve == false
+            raise Bolt::ValidationError,
+                  "Missing name for Git module specification: #{@git}. Git module specifications "\
+                  "must include a 'name' key when 'resolve' is false."
+          end
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+          end
         end
 
         def self.implements?(hash)
-          REQUIRED_KEYS == hash.keys.to_set
+          KNOWN_KEYS.superset?(hash.keys.to_set) && REQUIRED_KEYS.subset?(hash.keys.to_set)
         end
 
         # Parses the name into owner and name segments, and formats the full
@@ -47,6 +60,8 @@ module Bolt
         # Gets the repo from the git URL.
         #
         private def parse_git(git)
+          return [git, nil] unless @resolve
+
           repo = if git.start_with?('git@github.com:')
                    git.split('git@github.com:').last.split('.git').first
                  elsif git.start_with?('https://github.com')