bolt 2.36.0 → 2.42.0

data/lib/bolt/module_installer/resolver.rb

@@ -9,7 +9,7 @@ module Bolt
     class Resolver
       # Resolves module specs and returns a Puppetfile object.
       #
-      def resolve(specs)
+      def resolve(specs, config = {})
         require 'puppetfile-resolver'
 
         # Build the document model from the specs.
@@ -38,10 +38,10 @@ module Bolt
         # raised by puppetfile-resolver and re-raising them as Bolt errors.
         begin
           result = resolver.resolve(
-            cache:                 nil,
-            ui:                    nil,
-            module_paths:          [],
-            allow_missing_modules: false
+            cache:                       nil,
+            ui:                          nil,
+            allow_missing_modules:       false,
+            spec_searcher_configuration: spec_searcher_config(config)
           )
         rescue StandardError => e
           raise Bolt::Error.new(e.message, 'bolt/module-resolver-error')
@@ -71,6 +71,14 @@ module Bolt
         # Create the Puppetfile object.
         Bolt::ModuleInstaller::Puppetfile.new(modules)
       end
+
+      private def spec_searcher_config(config)
+        PuppetfileResolver::SpecSearchers::Configuration.new.tap do |obj|
+          obj.forge.proxy     = config.dig('forge', 'proxy') || config.dig('proxy')
+          obj.git.proxy       = config.dig('proxy')
+          obj.forge.forge_api = config.dig('forge', 'baseurl')
+        end
+      end
     end
   end
 end

data/lib/bolt/outputter.rb

@@ -2,24 +2,25 @@
 
 module Bolt
   class Outputter
-    def self.for_format(format, color, verbose, trace)
+    def self.for_format(format, color, verbose, trace, spin)
       case format
       when 'human'
-        Bolt::Outputter::Human.new(color, verbose, trace)
+        Bolt::Outputter::Human.new(color, verbose, trace, spin)
       when 'json'
-        Bolt::Outputter::JSON.new(color, verbose, trace)
+        Bolt::Outputter::JSON.new(color, verbose, trace, false)
       when 'rainbow'
-        Bolt::Outputter::Rainbow.new(color, verbose, trace)
+        Bolt::Outputter::Rainbow.new(color, verbose, trace, spin)
       when nil
         raise "Cannot use outputter before parsing."
       end
     end
 
-    def initialize(color, verbose, trace, stream = $stdout)
+    def initialize(color, verbose, trace, spin, stream = $stdout)
       @color = color
       @verbose = verbose
       @trace = trace
       @stream = stream
+      @spin = spin
     end
 
     def indent(indent, string)
@@ -34,6 +35,19 @@ module Bolt
     def print_error
       raise NotImplementedError, "print_error() must be implemented by the outputter class"
     end
+
+    def start_spin; end
+
+    def stop_spin; end
+
+    def spin
+      start_spin
+      begin
+        yield
+      ensure
+        stop_spin
+      end
+    end
   end
 end
 

data/lib/bolt/outputter/human.rb

@@ -14,12 +14,13 @@ module Bolt
 
       def print_head; end
 
-      def initialize(color, verbose, trace, stream = $stdout)
+      def initialize(color, verbose, trace, spin, stream = $stdout)
         super
         # Plans and without_default_logging() calls can both be nested, so we
         # track each of them with a "stack" consisting of an integer.
         @plan_depth = 0
         @disable_depth = 0
+        @pinwheel = %w[- \\ | /]
       end
 
       def colorize(color, string)
@@ -30,6 +31,24 @@ module Bolt
         end
       end
 
+      def start_spin
+        return unless @spin && @stream.isatty
+        @spin = true
+        @spin_thread = Thread.new do
+          loop do
+            sleep(0.1)
+            @stream.print(colorize(:cyan, @pinwheel.rotate!.first + "\b"))
+          end
+        end
+      end
+
+      def stop_spin
+        return unless @spin && @stream.isatty
+        @spin_thread.terminate
+        @spin = false
+        @stream.print("\b")
+      end
+
       def remove_trail(string)
         string.sub(/\s\z/, '')
       end

data/lib/bolt/outputter/json.rb

@@ -3,7 +3,7 @@
 module Bolt
   class Outputter
     class JSON < Bolt::Outputter
-      def initialize(color, verbose, trace, stream = $stdout)
+      def initialize(color, verbose, trace, spin, stream = $stdout)
         super
         @items_open = false
         @object_open = false

data/lib/bolt/outputter/logger.rb

@@ -6,7 +6,7 @@ module Bolt
   class Outputter
     class Logger < Bolt::Outputter
       def initialize(verbose, trace)
-        super(false, verbose, trace)
+        super(false, verbose, trace, false)
         @logger = Bolt::Logger.logger(self)
       end
 

data/lib/bolt/outputter/rainbow.rb

@@ -5,7 +5,7 @@ require 'bolt/pal'
 module Bolt
   class Outputter
     class Rainbow < Bolt::Outputter::Human
-      def initialize(color, verbose, trace, stream = $stdout)
+      def initialize(color, verbose, trace, spin, stream = $stdout)
         begin
           require 'paint'
           if Bolt::Util.windows?
@@ -62,6 +62,17 @@ module Bolt
         end
       end
 
+      def start_spin
+        return unless @spin && @stream.isatty
+        @spin = true
+        @spin_thread = Thread.new do
+          loop do
+            @stream.print(colorize(:rainbow, @pinwheel.rotate!.first + "\b"))
+            sleep(0.1)
+          end
+        end
+      end
+
       def print_summary(results, elapsed_time = nil)
         ok_set = results.ok_set
         unless ok_set.empty?

data/lib/bolt/pal/yaml_plan/transpiler.rb

@@ -64,7 +64,11 @@ module Bolt
             raise Bolt::FileError.new(msg, @plan_path)
           end
 
-          Bolt::PAL::YamlPlan::Loader.from_string(@modulename, file_contents, @plan_path)
+          begin
+            Bolt::PAL::YamlPlan::Loader.from_string(@modulename, file_contents, @plan_path)
+          rescue Puppet::PreformattedError, StandardError => e
+            raise PALError.from_preformatted_error(e)
+          end
         end
 
         def validate_path

data/lib/bolt/plugin.rb

@@ -4,6 +4,7 @@ require 'bolt/inventory'
 require 'bolt/executor'
 require 'bolt/module'
 require 'bolt/pal'
+require 'bolt/plugin/cache'
 require 'bolt/plugin/puppetdb'
 
 module Bolt
@@ -36,6 +37,13 @@ module Bolt
           super("Plugin #{plugin_name} does not support #{hook}", 'bolt/unsupported-hook')
         end
       end
+
+      class LoadingDisabled < PluginError
+        def initialize(plugin_name)
+          msg = "Cannot load plugin #{plugin_name}: plugin loading is disabled"
+          super(msg, 'bolt/plugin-loading-disabled', { 'plugin_name' => plugin_name })
+        end
+      end
     end
 
     class PluginContext
@@ -119,8 +127,8 @@ module Bolt
       end
     end
 
-    def self.setup(config, pal, analytics = Bolt::Analytics::NoopClient.new)
-      plugins = new(config, pal, analytics)
+    def self.setup(config, pal, analytics = Bolt::Analytics::NoopClient.new, **opts)
+      plugins = new(config, pal, analytics, **opts)
 
       config.plugins.each_key do |plugin|
         plugins.by_name(plugin)
@@ -141,12 +149,13 @@ module Bolt
 
     private_class_method :new
 
-    def initialize(config, pal, analytics)
+    def initialize(config, pal, analytics, load_plugins: true)
       @config = config
       @analytics = analytics
       @plugin_context = PluginContext.new(config, pal, self)
       @plugins = {}
       @pal = pal
+      @load_plugins = load_plugins
       @unknown = Set.new
       @resolution_stack = []
       @unresolved_plugin_configs = config.plugins.dup
@@ -161,7 +170,7 @@ module Bolt
     end
 
     def modules
-      @modules ||= Bolt::Module.discover(@pal.full_modulepath)
+      @modules ||= Bolt::Module.discover(@pal.full_modulepath, @config.project)
     end
 
     def add_plugin(plugin)
@@ -169,6 +178,8 @@ module Bolt
     end
 
     def add_ruby_plugin(plugin_name)
+      raise PluginError::LoadingDisabled, plugin_name unless @load_plugins
+
       cls_name = Bolt::Util.snake_name_to_class_name(plugin_name)
       filename = "bolt/plugin/#{plugin_name}"
       require filename
@@ -185,10 +196,17 @@ module Bolt
     def add_module_plugin(plugin_name)
       opts = {
         context: @plugin_context,
+        # Make sure that the plugin's config is validated _before_ the unknown-plugin
+        # and loading-disabled checks. This way, we can fail early on invalid plugin
+        # config instead of _after_ loading the modulepath (which can be expensive).
         config: config_for_plugin(plugin_name)
       }
 
-      plugin = Bolt::Plugin::Module.load(plugin_name, modules, opts)
+      mod = modules[plugin_name]
+      raise PluginError::Unknown, plugin_name unless mod&.plugin?
+      raise PluginError::LoadingDisabled, plugin_name unless @load_plugins
+
+      plugin = Bolt::Plugin::Module.load(mod, opts)
       add_plugin(plugin)
     end
 
@@ -277,6 +295,16 @@ module Bolt
     # Evaluates a single reference. The value returned may be another
     # reference.
     def resolve_single_reference(reference)
+      plugin_cache = if cache?(reference)
+                       cache = Bolt::Plugin::Cache.new(reference,
+                                                       @config.project.cache_file,
+                                                       @config.plugin_cache)
+                       entry = cache.read_and_clean_cache
+                       return entry unless entry.nil?
+
+                       cache
+                     end
+
       plugin_name = reference['_plugin']
       hook = get_hook(plugin_name, :resolve_reference)
 
@@ -288,16 +316,24 @@ module Bolt
 
       validate_proc.call(reference)
 
-      begin
+      result = begin
         # Evaluate the plugin and then recursively evaluate any plugin returned by it.
         hook.call(reference)
       rescue StandardError => e
         loc = "resolve_reference in #{plugin_name}"
         raise PluginError::ExecutionError.new(e.message, plugin_name, loc)
       end
+
+      plugin_cache.write_cache(result) if cache?(reference)
+
+      result
     end
     private :resolve_single_reference
 
+    private def cache?(reference)
+      reference.key?('_cache') || @config.plugin_cache.key?('ttl')
+    end
+
     # Checks whether a given value is a _plugin reference
     def reference?(input)
       input.is_a?(Hash) && input.key?('_plugin')

data/lib/bolt/plugin/cache.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'bolt/error'
+require 'bolt/util'
+
+module Bolt
+  class Plugin
+    class Cache
+      attr_reader :reference, :cache_file, :default_config, :id
+
+      def initialize(reference, cache_file, default_config)
+        @reference = reference
+        @cache_file = cache_file
+        @default_config = default_config
+      end
+
+      def read_and_clean_cache
+        return if ttl == 0
+        validate
+
+        # Luckily we don't need to use a serious hash algorithm
+        require 'digest/bubblebabble'
+        r = reference.reject { |k, _| k == '_cache' }.sort.to_s
+        @id = Digest::SHA2.bubblebabble(r)[0..20]
+
+        unmodified = true
+        # First remove any cache entries past their ttl
+        # This prevents removing plugins from leaving orphaned cache entries
+        cache.delete_if do |_, entry|
+          expired = Time.now - Time.parse(entry['mtime']) >= entry['ttl']
+          unmodified = false if expired
+          expired
+        end
+        File.write(cache_file, cache.to_json) unless cache.empty? || unmodified
+
+        cache.dig(id, 'result')
+      end
+
+      private def cache
+        @cache ||= Bolt::Util.read_optional_json_file(@cache_file, 'cache')
+      end
+
+      def write_cache(result)
+        cache.merge!({ id => { 'result' => result,
+                               'mtime' => Time.now,
+                               'ttl' => ttl } })
+        FileUtils.touch(cache_file)
+        File.write(cache_file, cache.to_json)
+      end
+
+      def validate
+        # The default cache `plugin-cache` will be validated by the config
+        # validator
+        return if reference['_cache'].nil?
+        r = reference['_cache']
+        unless r.is_a?(Hash)
+          raise Bolt::ValidationError,
+                "_cache must be a Hash, received #{r.class}: #{r.inspect}"
+        end
+
+        unless r.key?('ttl')
+          raise Bolt::ValidationError, "_cache must set 'ttl' key."
+        end
+
+        unless r['ttl'] >= 0
+          raise Bolt::ValidationError, "'ttl' key under '_cache' must be a minimum of 0."
+        end
+      end
+
+      private def ttl
+        @ttl ||= reference.dig('_cache', 'ttl') || default_config['ttl']
+      end
+    end
+  end
+end

data/lib/bolt/plugin/module.rb

@@ -12,15 +12,15 @@ module Bolt
         end
       end
 
-      def self.load(name, modules, opts)
-        mod = modules[name]
-        if mod&.plugin?
+      # mod should not be nil
+      def self.load(mod, opts)
+        if mod.plugin?
           opts[:mod] = mod
           plugin = Bolt::Plugin::Module.new(**opts)
           plugin.setup
           plugin
         else
-          raise PluginError::Unknown, name
+          raise PluginError::Unknown, mod.name
         end
       end
 

data/lib/bolt/plugin/puppetdb.rb

@@ -12,7 +12,7 @@ module Bolt
       end
 
       TEMPLATE_OPTS = %w[alias config facts features name uri vars].freeze
-      PLUGIN_OPTS = %w[_plugin query target_mapping].freeze
+      PLUGIN_OPTS = %w[_plugin _cache query target_mapping].freeze
 
       attr_reader :puppetdb_client
 

data/lib/bolt/project.rb

@@ -2,7 +2,7 @@
 
 require 'pathname'
 require 'bolt/config'
-require 'bolt/config/validator'
+require 'bolt/validator'
 require 'bolt/pal'
 require 'bolt/module'
 
@@ -14,7 +14,7 @@ module Bolt
     attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
                 :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
-                :backup_dir
+                :backup_dir, :cache_file
 
     def self.default_project(logs = [])
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
@@ -27,23 +27,31 @@ module Bolt
     # directory called Boltdir or a file called bolt.yaml (for a control repo
     # type Project). Otherwise, repeat the check on each directory up the
     # hierarchy, falling back to the default if we reach the root.
-    def self.find_boltdir(dir, logs = [])
+    def self.find_boltdir(dir, logs = [], deprecations = [])
       dir = Pathname.new(dir)
 
       if (dir + BOLTDIR_NAME).directory?
         create_project(dir + BOLTDIR_NAME, 'embedded', logs)
-      elsif (dir + 'bolt.yaml').file? || (dir + CONFIG_NAME).file?
+      elsif (dir + 'bolt.yaml').file?
+        command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
+        msg = "Configuration file #{dir + 'bolt.yaml'} is deprecated and will be "\
+          "removed in Bolt 3.0.\nUpdate your Bolt project to the latest Bolt practices "\
+          "using #{command}"
+        deprecations << { type: "Project level bolt.yaml",
+                          msg: msg }
+        create_project(dir, 'local', logs, deprecations)
+      elsif (dir + CONFIG_NAME).file?
         create_project(dir, 'local', logs)
       elsif dir.root?
         default_project(logs)
       else
         logs << { debug: "Did not detect Boltdir, bolt.yaml, or bolt-project.yaml at '#{dir}'. "\
                   "This directory won't be loaded as a project." }
-        find_boltdir(dir.parent, logs)
+        find_boltdir(dir.parent, logs, deprecations)
       end
     end
 
-    def self.create_project(path, type = 'option', logs = [])
+    def self.create_project(path, type = 'option', logs = [], deprecations = [])
       fullpath = Pathname.new(path).expand_path
 
       if type == 'user'
@@ -72,15 +80,10 @@ module Bolt
       data         = Bolt::Util.read_optional_yaml_hash(File.expand_path(project_file), 'project')
       default      = type =~ /user|system/ ? 'default ' : ''
       exist        = File.exist?(File.expand_path(project_file))
-      deprecations = []
 
       logs << { info: "Loaded #{default}project from '#{fullpath}'" } if exist
 
-      # Validate the config against the schema. This will raise a single error
-      # with all validation errors.
-      schema = Bolt::Config::OPTIONS.slice(*Bolt::Config::BOLT_PROJECT_OPTIONS)
-
-      Bolt::Config::Validator.new.tap do |validator|
+      Bolt::Validator.new.tap do |validator|
         validator.validate(data, schema, project_file)
 
         validator.warnings.each { |warning| logs << { warn: warning } }
@@ -93,6 +96,16 @@ module Bolt
       new(data, path, type, logs, deprecations)
     end
 
+    # Builds the schema for bolt-project.yaml used by the validator.
+    #
+    def self.schema
+      {
+        type:        Hash,
+        properties:  Bolt::Config::BOLT_PROJECT_OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: Bolt::Config::OPTIONS
+      }
+    end
+
     def initialize(raw_data, path, type = 'option', logs = [], deprecations = [])
       @path         = Pathname.new(path).expand_path
       @project_file = @path + CONFIG_NAME
@@ -116,6 +129,7 @@ module Bolt
       @plans_path        = @path + 'plans'
       @managed_moduledir = @path + '.modules'
       @backup_dir        = @path + '.bolt-bak'
+      @cache_file        = @path + '.plugin_cache.json'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
@@ -184,6 +198,14 @@ module Bolt
       @data['plans']
     end
 
+    def plugin_cache
+      @data['plugin-cache']
+    end
+
+    def module_install
+      @data['module-install']
+    end
+
     def modules
       @modules ||= @data['modules']&.map do |mod|
         if mod.is_a?(String)
@@ -205,7 +227,10 @@ module Bolt
           raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
             "with a built-in Bolt module of the same name."
         end
-      else
+      elsif name.nil? &&
+            (File.directory?(plans_path) ||
+            File.directory?(@path + 'tasks') ||
+            File.directory?(@path + 'files'))
         message = "No project name is specified in bolt-project.yaml. Project-level content will not be available."
         @logs << { warn: message }
       end