bolt 2.35.0 → 2.40.2

data/lib/bolt/inventory/target.rb

@@ -31,7 +31,8 @@ module Bolt
         end
 
         if @name == 'localhost'
-          target_data = localhost_defaults(target_data)
+          default = { 'config' => { 'transport' => 'local' } }
+          target_data = Bolt::Util.deep_merge(default, target_data)
         end
 
         @config = target_data['config'] || {}
@@ -49,18 +50,16 @@ module Bolt
         validate
       end
 
-      def localhost_defaults(data)
+      def set_local_defaults
+        return if @set_local_default
         defaults = {
-          'config' => {
-            'transport' => 'local',
-            'local' => { 'interpreters' => { '.rb' => RbConfig.ruby } }
-          },
-          'features' => ['puppet-agent']
+          'local' => { 'interpreters' => { '.rb' => RbConfig.ruby } }
         }
-        data = Bolt::Util.deep_merge(defaults, data)
-        # If features is an empty array deep_merge won't add the puppet-agent
-        data['features'] += ['puppet-agent'] if data['features'].empty?
-        data
+        old_config = @config
+        @config = Bolt::Util.deep_merge(defaults, @config)
+        invalidate_config_cache! if old_config != @config
+        set_feature('puppet-agent')
+        @set_local_default = true
       end
 
       # rubocop:disable Naming/AccessorMethodName

data/lib/bolt/module_installer.rb

@@ -17,14 +17,14 @@ module Bolt
 
     # Adds a single module to the project.
     #
-    def add(name, specs, puppetfile_path, moduledir, config_path)
+    def add(name, specs, puppetfile_path, moduledir, project_file, config)
       project_specs = Specs.new(specs)
 
       # Exit early if project config already includes a spec with this name.
       if project_specs.include?(name)
         @outputter.print_message(
-          "Project configuration file #{config_path} already includes specification with name "\
-          "#{name}. Nothing to do."
+          "Project configuration file #{project_file} already includes specification "\
+          "with name #{name}. Nothing to do."
         )
         return true
       end
@@ -47,30 +47,32 @@ module Bolt
       # a version conflict.
       @outputter.print_action_step("Resolving module dependencies, this may take a moment")
 
+      @outputter.start_spin
       begin
         resolve_specs.add_specs('name' => name)
-        puppetfile = Resolver.new.resolve(resolve_specs)
+        puppetfile = Resolver.new.resolve(resolve_specs, config)
       rescue Bolt::Error
         project_specs.add_specs('name' => name)
-        puppetfile = Resolver.new.resolve(project_specs)
+        puppetfile = Resolver.new.resolve(project_specs, config)
       end
+      @outputter.stop_spin
 
       # Display the diff between the existing Puppetfile and the new Puppetfile.
       print_puppetfile_diff(existing_puppetfile, puppetfile)
 
       # Add the module to the project configuration.
-      @outputter.print_action_step("Updating project configuration file at #{config_path}")
+      @outputter.print_action_step("Updating project configuration file at #{project_file}")
 
-      data = Bolt::Util.read_yaml_hash(config_path, 'project')
+      data = Bolt::Util.read_yaml_hash(project_file, 'project')
       data['modules'] ||= []
       data['modules'] << name.tr('-', '/')
 
       begin
-        File.write(config_path, data.to_yaml)
+        File.write(project_file, data.to_yaml)
       rescue SystemCallError => e
         raise Bolt::FileError.new(
           "Unable to update project configuration file: #{e.message}",
-          config
+          project_file
         )
       end
 
@@ -79,7 +81,7 @@ module Bolt
       puppetfile.write(puppetfile_path, moduledir)
 
       # Install the modules.
-      install_puppetfile(puppetfile_path, moduledir)
+      install_puppetfile(puppetfile_path, moduledir, config)
     end
 
     # Outputs a diff of an old Puppetfile and a new Puppetfile.
@@ -145,7 +147,7 @@ module Bolt
 
     # Installs a project's module dependencies.
     #
-    def install(specs, path, moduledir, force: false, resolve: true)
+    def install(specs, path, moduledir, config = {}, force: false, resolve: true)
       @outputter.print_message("Installing project modules\n\n")
 
       if resolve != false
@@ -155,7 +157,11 @@ module Bolt
         # and write a Puppetfile.
         if force || !path.exist?
           @outputter.print_action_step("Resolving module dependencies, this may take a moment")
-          puppetfile = Resolver.new.resolve(specs)
+
+          # This doesn't use the block as it's more testable to just mock *_spin
+          @outputter.start_spin
+          puppetfile = Resolver.new.resolve(specs, config)
+          @outputter.stop_spin
 
           # We get here either through 'bolt module install' which uses the
           # managed modulepath (which isn't configurable) or through bolt
@@ -177,14 +183,16 @@ module Bolt
       end
 
       # Install the modules.
-      install_puppetfile(path, moduledir)
+      install_puppetfile(path, moduledir, config)
     end
 
     # Installs the Puppetfile and generates types.
     #
     def install_puppetfile(path, moduledir, config = {})
       @outputter.print_action_step("Syncing modules from #{path} to #{moduledir}")
+      @outputter.start_spin
       ok = Installer.new(config).install(path, moduledir)
+      @outputter.stop_spin
 
       # Automatically generate types after installing modules
       @outputter.print_action_step("Generating type references")

data/lib/bolt/module_installer/resolver.rb

@@ -9,7 +9,7 @@ module Bolt
     class Resolver
       # Resolves module specs and returns a Puppetfile object.
       #
-      def resolve(specs)
+      def resolve(specs, _config = {})
         require 'puppetfile-resolver'
 
         # Build the document model from the specs.

data/lib/bolt/outputter.rb

@@ -2,24 +2,25 @@
 
 module Bolt
   class Outputter
-    def self.for_format(format, color, verbose, trace)
+    def self.for_format(format, color, verbose, trace, spin)
       case format
       when 'human'
-        Bolt::Outputter::Human.new(color, verbose, trace)
+        Bolt::Outputter::Human.new(color, verbose, trace, spin)
       when 'json'
-        Bolt::Outputter::JSON.new(color, verbose, trace)
+        Bolt::Outputter::JSON.new(color, verbose, trace, false)
       when 'rainbow'
-        Bolt::Outputter::Rainbow.new(color, verbose, trace)
+        Bolt::Outputter::Rainbow.new(color, verbose, trace, spin)
       when nil
         raise "Cannot use outputter before parsing."
       end
     end
 
-    def initialize(color, verbose, trace, stream = $stdout)
+    def initialize(color, verbose, trace, spin, stream = $stdout)
       @color = color
       @verbose = verbose
       @trace = trace
       @stream = stream
+      @spin = spin
     end
 
     def indent(indent, string)
@@ -34,6 +35,19 @@ module Bolt
     def print_error
       raise NotImplementedError, "print_error() must be implemented by the outputter class"
     end
+
+    def start_spin; end
+
+    def stop_spin; end
+
+    def spin
+      start_spin
+      begin
+        yield
+      ensure
+        stop_spin
+      end
+    end
   end
 end
 

data/lib/bolt/outputter/human.rb

@@ -14,12 +14,13 @@ module Bolt
 
       def print_head; end
 
-      def initialize(color, verbose, trace, stream = $stdout)
+      def initialize(color, verbose, trace, spin, stream = $stdout)
         super
         # Plans and without_default_logging() calls can both be nested, so we
         # track each of them with a "stack" consisting of an integer.
         @plan_depth = 0
         @disable_depth = 0
+        @pinwheel = %w[- \\ | /]
       end
 
       def colorize(color, string)
@@ -30,6 +31,24 @@ module Bolt
         end
       end
 
+      def start_spin
+        return unless @spin && @stream.isatty
+        @spin = true
+        @spin_thread = Thread.new do
+          loop do
+            sleep(0.1)
+            @stream.print(colorize(:cyan, @pinwheel.rotate!.first + "\b"))
+          end
+        end
+      end
+
+      def stop_spin
+        return unless @spin && @stream.isatty
+        @spin_thread.terminate
+        @spin = false
+        @stream.print("\b")
+      end
+
       def remove_trail(string)
         string.sub(/\s\z/, '')
       end

data/lib/bolt/outputter/json.rb

@@ -3,7 +3,7 @@
 module Bolt
   class Outputter
     class JSON < Bolt::Outputter
-      def initialize(color, verbose, trace, stream = $stdout)
+      def initialize(color, verbose, trace, spin, stream = $stdout)
         super
         @items_open = false
         @object_open = false

data/lib/bolt/outputter/logger.rb

@@ -6,7 +6,7 @@ module Bolt
   class Outputter
     class Logger < Bolt::Outputter
       def initialize(verbose, trace)
-        super(false, verbose, trace)
+        super(false, verbose, trace, false)
         @logger = Bolt::Logger.logger(self)
       end
 

data/lib/bolt/outputter/rainbow.rb

@@ -5,7 +5,7 @@ require 'bolt/pal'
 module Bolt
   class Outputter
     class Rainbow < Bolt::Outputter::Human
-      def initialize(color, verbose, trace, stream = $stdout)
+      def initialize(color, verbose, trace, spin, stream = $stdout)
         begin
           require 'paint'
           if Bolt::Util.windows?
@@ -53,7 +53,7 @@ module Bolt
                 @state = :normal if c == 'm'
               end
             end
-            a.join('')
+            a.join
           else
             "\033[#{COLORS[color]}m#{string}\033[0m"
           end
@@ -62,6 +62,17 @@ module Bolt
         end
       end
 
+      def start_spin
+        return unless @spin && @stream.isatty
+        @spin = true
+        @spin_thread = Thread.new do
+          loop do
+            @stream.print(colorize(:rainbow, @pinwheel.rotate!.first + "\b"))
+            sleep(0.1)
+          end
+        end
+      end
+
       def print_summary(results, elapsed_time = nil)
         ok_set = results.ok_set
         unless ok_set.empty?

data/lib/bolt/plugin.rb

@@ -4,6 +4,7 @@ require 'bolt/inventory'
 require 'bolt/executor'
 require 'bolt/module'
 require 'bolt/pal'
+require 'bolt/plugin/cache'
 require 'bolt/plugin/puppetdb'
 
 module Bolt
@@ -36,6 +37,13 @@ module Bolt
           super("Plugin #{plugin_name} does not support #{hook}", 'bolt/unsupported-hook')
         end
       end
+
+      class LoadingDisabled < PluginError
+        def initialize(plugin_name)
+          msg = "Cannot load plugin #{plugin_name}: plugin loading is disabled"
+          super(msg, 'bolt/plugin-loading-disabled', { 'plugin_name' => plugin_name })
+        end
+      end
     end
 
     class PluginContext
@@ -119,15 +127,8 @@ module Bolt
       end
     end
 
-    def self.setup(config, pal, analytics = Bolt::Analytics::NoopClient.new)
-      plugins = new(config, pal, analytics)
-
-      # Initialize any plugins referenced in plugin config. This will also indirectly
-      # initialize any plugins they depend on.
-      if plugins.reference?(config.plugins)
-        msg = "The 'plugins' setting cannot be set by a plugin reference"
-        raise PluginError.new(msg, 'bolt/plugin-error')
-      end
+    def self.setup(config, pal, analytics = Bolt::Analytics::NoopClient.new, **opts)
+      plugins = new(config, pal, analytics, **opts)
 
       config.plugins.each_key do |plugin|
         plugins.by_name(plugin)
@@ -148,12 +149,13 @@ module Bolt
 
     private_class_method :new
 
-    def initialize(config, pal, analytics)
+    def initialize(config, pal, analytics, load_plugins: true)
       @config = config
       @analytics = analytics
       @plugin_context = PluginContext.new(config, pal, self)
       @plugins = {}
       @pal = pal
+      @load_plugins = load_plugins
       @unknown = Set.new
       @resolution_stack = []
       @unresolved_plugin_configs = config.plugins.dup
@@ -176,6 +178,8 @@ module Bolt
     end
 
     def add_ruby_plugin(plugin_name)
+      raise PluginError::LoadingDisabled, plugin_name unless @load_plugins
+
       cls_name = Bolt::Util.snake_name_to_class_name(plugin_name)
       filename = "bolt/plugin/#{plugin_name}"
       require filename
@@ -192,10 +196,17 @@ module Bolt
     def add_module_plugin(plugin_name)
       opts = {
         context: @plugin_context,
+        # Make sure that the plugin's config is validated _before_ the unknown-plugin
+        # and loading-disabled checks. This way, we can fail early on invalid plugin
+        # config instead of _after_ loading the modulepath (which can be expensive).
         config: config_for_plugin(plugin_name)
       }
 
-      plugin = Bolt::Plugin::Module.load(plugin_name, modules, opts)
+      mod = modules[plugin_name]
+      raise PluginError::Unknown, plugin_name unless mod&.plugin?
+      raise PluginError::LoadingDisabled, plugin_name unless @load_plugins
+
+      plugin = Bolt::Plugin::Module.load(mod, opts)
       add_plugin(plugin)
     end
 
@@ -284,6 +295,16 @@ module Bolt
     # Evaluates a single reference. The value returned may be another
     # reference.
     def resolve_single_reference(reference)
+      plugin_cache = if cache?(reference)
+                       cache = Bolt::Plugin::Cache.new(reference,
+                                                       @config.project.cache_file,
+                                                       @config.plugin_cache)
+                       entry = cache.read_and_clean_cache
+                       return entry unless entry.nil?
+
+                       cache
+                     end
+
       plugin_name = reference['_plugin']
       hook = get_hook(plugin_name, :resolve_reference)
 
@@ -295,16 +316,24 @@ module Bolt
 
       validate_proc.call(reference)
 
-      begin
+      result = begin
         # Evaluate the plugin and then recursively evaluate any plugin returned by it.
         hook.call(reference)
       rescue StandardError => e
         loc = "resolve_reference in #{plugin_name}"
         raise PluginError::ExecutionError.new(e.message, plugin_name, loc)
       end
+
+      plugin_cache.write_cache(result) if cache?(reference)
+
+      result
     end
     private :resolve_single_reference
 
+    private def cache?(reference)
+      reference.key?('_cache') || @config.plugin_cache.key?('ttl')
+    end
+
     # Checks whether a given value is a _plugin reference
     def reference?(input)
       input.is_a?(Hash) && input.key?('_plugin')

data/lib/bolt/plugin/cache.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'bolt/error'
+require 'bolt/util'
+
+module Bolt
+  class Plugin
+    class Cache
+      attr_reader :reference, :cache_file, :default_config, :id
+
+      def initialize(reference, cache_file, default_config)
+        @reference = reference
+        @cache_file = cache_file
+        @default_config = default_config
+      end
+
+      def read_and_clean_cache
+        return if ttl == 0
+        validate
+
+        # Luckily we don't need to use a serious hash algorithm
+        require 'digest/bubblebabble'
+        r = reference.reject { |k, _| k == '_cache' }.sort.to_s
+        @id = Digest::SHA2.bubblebabble(r)[0..20]
+
+        unmodified = true
+        # First remove any cache entries past their ttl
+        # This prevents removing plugins from leaving orphaned cache entries
+        cache.delete_if do |_, entry|
+          expired = Time.now - Time.parse(entry['mtime']) >= entry['ttl']
+          unmodified = false if expired
+          expired
+        end
+        File.write(cache_file, cache.to_json) unless cache.empty? || unmodified
+
+        cache.dig(id, 'result')
+      end
+
+      private def cache
+        @cache ||= Bolt::Util.read_optional_json_file(@cache_file, 'cache')
+      end
+
+      def write_cache(result)
+        cache.merge!({ id => { 'result' => result,
+                               'mtime' => Time.now,
+                               'ttl' => ttl } })
+        FileUtils.touch(cache_file)
+        File.write(cache_file, cache.to_json)
+      end
+
+      def validate
+        # The default cache `plugin-cache` will be validated by the config
+        # validator
+        return if reference['_cache'].nil?
+        r = reference['_cache']
+        unless r.is_a?(Hash)
+          raise Bolt::ValidationError,
+                "_cache must be a Hash, received #{r.class}: #{r.inspect}"
+        end
+
+        unless r.key?('ttl')
+          raise Bolt::ValidationError, "_cache must set 'ttl' key."
+        end
+
+        unless r['ttl'] >= 0
+          raise Bolt::ValidationError, "'ttl' key under '_cache' must be a minimum of 0."
+        end
+      end
+
+      private def ttl
+        @ttl ||= reference.dig('_cache', 'ttl') || default_config['ttl']
+      end
+    end
+  end
+end