bolt 2.34.0 → 2.40.1

data/lib/bolt/pal.rb

@@ -26,7 +26,7 @@ module Bolt
         details[:line]   = err.line if defined?(err.line)
         details[:column] = err.pos if defined?(err.pos)
 
-        error.add_filelineno(details)
+        error.add_filelineno(details.compact)
         error
       end
 
@@ -286,15 +286,26 @@ module Bolt
       raise Bolt::PAL::PALError, "Failed to parse manifest: #{e}"
     end
 
-    def list_tasks
+    # Filters content by a list of names and glob patterns specified in project
+    # configuration.
+    def filter_content(content, patterns)
+      return content unless content && patterns
+
+      content.select do |name,|
+        patterns.any? { |pattern| File.fnmatch?(pattern, name, File::FNM_EXTGLOB) }
+      end
+    end
+
+    def list_tasks(filter_content: false)
       in_bolt_compiler do |compiler|
-        tasks = compiler.list_tasks
-        tasks.map(&:name).sort.each_with_object([]) do |task_name, data|
+        tasks = compiler.list_tasks.map(&:name).sort.each_with_object([]) do |task_name, data|
           task_sig = compiler.task_signature(task_name)
           unless task_sig.task_hash['metadata']['private']
             data << [task_name, task_sig.task_hash['metadata']['description']]
           end
         end
+
+        filter_content ? filter_content(tasks, @project&.tasks) : tasks
       end
     end
 
@@ -346,14 +357,15 @@ module Bolt
       Bolt::Task.from_task_signature(task)
     end
 
-    def list_plans
+    def list_plans(filter_content: false)
       in_bolt_compiler do |compiler|
         errors = []
         plans = compiler.list_plans(nil, errors).map { |plan| [plan.name] }.sort
         errors.each do |error|
           @logger.warn(error.details['original_error'])
         end
-        plans
+
+        filter_content ? filter_content(plans, @project&.plans) : plans
       end
     end
 

data/lib/bolt/pal/yaml_plan.rb

@@ -45,6 +45,13 @@ module Bolt
         used_names = Set.new(@parameters.map(&:name))
 
         @steps = plan['steps'].each_with_index.map do |step, index|
+          unless step.is_a?(Hash)
+            raise Bolt::Error.new(
+              "Parse error in step number #{index + 1}: Plan step must be an object with valid step keys.",
+              'bolt/invalid-plan'
+            )
+          end
+
           # Step keys also aren't allowed to be code and neither is the value of "name"
           stringified_step = Bolt::Util.walk_keys(step) { |key| stringify(key) }
           stringified_step['name'] = stringify(stringified_step['name']) if stringified_step.key?('name')

data/lib/bolt/plugin.rb

@@ -4,6 +4,7 @@ require 'bolt/inventory'
 require 'bolt/executor'
 require 'bolt/module'
 require 'bolt/pal'
+require 'bolt/plugin/cache'
 require 'bolt/plugin/puppetdb'
 
 module Bolt
@@ -36,6 +37,13 @@ module Bolt
           super("Plugin #{plugin_name} does not support #{hook}", 'bolt/unsupported-hook')
         end
       end
+
+      class LoadingDisabled < PluginError
+        def initialize(plugin_name)
+          msg = "Cannot load plugin #{plugin_name}: plugin loading is disabled"
+          super(msg, 'bolt/plugin-loading-disabled', { 'plugin_name' => plugin_name })
+        end
+      end
     end
 
     class PluginContext
@@ -119,15 +127,8 @@ module Bolt
       end
     end
 
-    def self.setup(config, pal, analytics = Bolt::Analytics::NoopClient.new)
-      plugins = new(config, pal, analytics)
-
-      # Initialize any plugins referenced in plugin config. This will also indirectly
-      # initialize any plugins they depend on.
-      if plugins.reference?(config.plugins)
-        msg = "The 'plugins' setting cannot be set by a plugin reference"
-        raise PluginError.new(msg, 'bolt/plugin-error')
-      end
+    def self.setup(config, pal, analytics = Bolt::Analytics::NoopClient.new, **opts)
+      plugins = new(config, pal, analytics, **opts)
 
       config.plugins.each_key do |plugin|
         plugins.by_name(plugin)
@@ -148,12 +149,13 @@ module Bolt
 
     private_class_method :new
 
-    def initialize(config, pal, analytics)
+    def initialize(config, pal, analytics, load_plugins: true)
       @config = config
       @analytics = analytics
       @plugin_context = PluginContext.new(config, pal, self)
       @plugins = {}
       @pal = pal
+      @load_plugins = load_plugins
       @unknown = Set.new
       @resolution_stack = []
       @unresolved_plugin_configs = config.plugins.dup
@@ -176,6 +178,8 @@ module Bolt
     end
 
     def add_ruby_plugin(plugin_name)
+      raise PluginError::LoadingDisabled, plugin_name unless @load_plugins
+
       cls_name = Bolt::Util.snake_name_to_class_name(plugin_name)
       filename = "bolt/plugin/#{plugin_name}"
       require filename
@@ -192,10 +196,17 @@ module Bolt
     def add_module_plugin(plugin_name)
       opts = {
         context: @plugin_context,
+        # Make sure that the plugin's config is validated _before_ the unknown-plugin
+        # and loading-disabled checks. This way, we can fail early on invalid plugin
+        # config instead of _after_ loading the modulepath (which can be expensive).
         config: config_for_plugin(plugin_name)
       }
 
-      plugin = Bolt::Plugin::Module.load(plugin_name, modules, opts)
+      mod = modules[plugin_name]
+      raise PluginError::Unknown, plugin_name unless mod&.plugin?
+      raise PluginError::LoadingDisabled, plugin_name unless @load_plugins
+
+      plugin = Bolt::Plugin::Module.load(mod, opts)
       add_plugin(plugin)
     end
 
@@ -284,6 +295,16 @@ module Bolt
     # Evaluates a single reference. The value returned may be another
     # reference.
     def resolve_single_reference(reference)
+      plugin_cache = if cache?(reference)
+                       cache = Bolt::Plugin::Cache.new(reference,
+                                                       @config.project.cache_file,
+                                                       @config.plugin_cache)
+                       entry = cache.read_and_clean_cache
+                       return entry unless entry.nil?
+
+                       cache
+                     end
+
       plugin_name = reference['_plugin']
       hook = get_hook(plugin_name, :resolve_reference)
 
@@ -295,16 +316,24 @@ module Bolt
 
       validate_proc.call(reference)
 
-      begin
+      result = begin
         # Evaluate the plugin and then recursively evaluate any plugin returned by it.
         hook.call(reference)
       rescue StandardError => e
         loc = "resolve_reference in #{plugin_name}"
         raise PluginError::ExecutionError.new(e.message, plugin_name, loc)
       end
+
+      plugin_cache.write_cache(result) if cache?(reference)
+
+      result
     end
     private :resolve_single_reference
 
+    private def cache?(reference)
+      reference.key?('_cache') || @config.plugin_cache.key?('ttl')
+    end
+
     # Checks whether a given value is a _plugin reference
     def reference?(input)
       input.is_a?(Hash) && input.key?('_plugin')

data/lib/bolt/plugin/cache.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'bolt/error'
+require 'bolt/util'
+
+module Bolt
+  class Plugin
+    class Cache
+      attr_reader :reference, :cache_file, :default_config, :id
+
+      def initialize(reference, cache_file, default_config)
+        @reference = reference
+        @cache_file = cache_file
+        @default_config = default_config
+      end
+
+      def read_and_clean_cache
+        return if ttl == 0
+        validate
+
+        # Luckily we don't need to use a serious hash algorithm
+        require 'digest/bubblebabble'
+        r = reference.reject { |k, _| k == '_cache' }.sort.to_s
+        @id = Digest::SHA2.bubblebabble(r)[0..20]
+
+        unmodified = true
+        # First remove any cache entries past their ttl
+        # This prevents removing plugins from leaving orphaned cache entries
+        cache.delete_if do |_, entry|
+          expired = Time.now - Time.parse(entry['mtime']) >= entry['ttl']
+          unmodified = false if expired
+          expired
+        end
+        File.write(cache_file, cache.to_json) unless cache.empty? || unmodified
+
+        cache.dig(id, 'result')
+      end
+
+      private def cache
+        @cache ||= Bolt::Util.read_optional_json_file(@cache_file, 'cache')
+      end
+
+      def write_cache(result)
+        cache.merge!({ id => { 'result' => result,
+                               'mtime' => Time.now,
+                               'ttl' => ttl } })
+        FileUtils.touch(cache_file)
+        File.write(cache_file, cache.to_json)
+      end
+
+      def validate
+        # The default cache `plugin-cache` will be validated by the config
+        # validator
+        return if reference['_cache'].nil?
+        r = reference['_cache']
+        unless r.is_a?(Hash)
+          raise Bolt::ValidationError,
+                "_cache must be a Hash, received #{r.class}: #{r.inspect}"
+        end
+
+        unless r.key?('ttl')
+          raise Bolt::ValidationError, "_cache must set 'ttl' key."
+        end
+
+        unless r['ttl'] >= 0
+          raise Bolt::ValidationError, "'ttl' key under '_cache' must be a minimum of 0."
+        end
+      end
+
+      private def ttl
+        @ttl ||= reference.dig('_cache', 'ttl') || default_config['ttl']
+      end
+    end
+  end
+end

data/lib/bolt/plugin/module.rb

@@ -12,15 +12,15 @@ module Bolt
         end
       end
 
-      def self.load(name, modules, opts)
-        mod = modules[name]
-        if mod&.plugin?
+      # mod should not be nil
+      def self.load(mod, opts)
+        if mod.plugin?
           opts[:mod] = mod
           plugin = Bolt::Plugin::Module.new(**opts)
           plugin.setup
           plugin
         else
-          raise PluginError::Unknown, name
+          raise PluginError::Unknown, mod.name
         end
       end
 

data/lib/bolt/plugin/puppetdb.rb

@@ -12,7 +12,7 @@ module Bolt
       end
 
       TEMPLATE_OPTS = %w[alias config facts features name uri vars].freeze
-      PLUGIN_OPTS = %w[_plugin query target_mapping].freeze
+      PLUGIN_OPTS = %w[_plugin _cache query target_mapping].freeze
 
       attr_reader :puppetdb_client
 

data/lib/bolt/project.rb

@@ -2,24 +2,19 @@
 
 require 'pathname'
 require 'bolt/config'
+require 'bolt/validator'
 require 'bolt/pal'
 require 'bolt/module'
 
 module Bolt
   class Project
     BOLTDIR_NAME = 'Boltdir'
-    PROJECT_SETTINGS = {
-      "name"  => "The name of the project",
-      "plans" => "An array of plan names to show, if they exist in the project."\
-                 "These plans are included in `bolt plan show` and `Get-BoltPlan` output",
-      "tasks" => "An array of task names to show, if they exist in the project."\
-                 "These tasks are included in `bolt task show` and `Get-BoltTask` output"
-    }.freeze
+    CONFIG_NAME  = 'bolt-project.yaml'
 
     attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
                 :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
-                :backup_dir
+                :backup_dir, :cache_file
 
     def self.default_project(logs = [])
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
@@ -32,23 +27,31 @@ module Bolt
     # directory called Boltdir or a file called bolt.yaml (for a control repo
     # type Project). Otherwise, repeat the check on each directory up the
     # hierarchy, falling back to the default if we reach the root.
-    def self.find_boltdir(dir, logs = [])
+    def self.find_boltdir(dir, logs = [], deprecations = [])
       dir = Pathname.new(dir)
 
       if (dir + BOLTDIR_NAME).directory?
         create_project(dir + BOLTDIR_NAME, 'embedded', logs)
-      elsif (dir + 'bolt.yaml').file? || (dir + 'bolt-project.yaml').file?
+      elsif (dir + 'bolt.yaml').file?
+        command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
+        msg = "Configuration file #{dir + 'bolt.yaml'} is deprecated and will be "\
+          "removed in Bolt 3.0.\nUpdate your Bolt project to the latest Bolt practices "\
+          "using #{command}"
+        deprecations << { type: "Project level bolt.yaml",
+                          msg: msg }
+        create_project(dir, 'local', logs, deprecations)
+      elsif (dir + CONFIG_NAME).file?
         create_project(dir, 'local', logs)
       elsif dir.root?
         default_project(logs)
       else
         logs << { debug: "Did not detect Boltdir, bolt.yaml, or bolt-project.yaml at '#{dir}'. "\
                   "This directory won't be loaded as a project." }
-        find_boltdir(dir.parent, logs)
+        find_boltdir(dir.parent, logs, deprecations)
       end
     end
 
-    def self.create_project(path, type = 'option', logs = [])
+    def self.create_project(path, type = 'option', logs = [], deprecations = [])
       fullpath = Pathname.new(path).expand_path
 
       if type == 'user'
@@ -73,21 +76,42 @@ module Bolt
         )
       end
 
-      project_file = File.join(fullpath, 'bolt-project.yaml')
-      data = Bolt::Util.read_optional_yaml_hash(File.expand_path(project_file), 'project')
-      default = type =~ /user|system/ ? 'default ' : ''
-      exist = File.exist?(File.expand_path(project_file))
+      project_file = File.join(fullpath, CONFIG_NAME)
+      data         = Bolt::Util.read_optional_yaml_hash(File.expand_path(project_file), 'project')
+      default      = type =~ /user|system/ ? 'default ' : ''
+      exist        = File.exist?(File.expand_path(project_file))
+
       logs << { info: "Loaded #{default}project from '#{fullpath}'" } if exist
-      new(data, path, type, logs)
+
+      Bolt::Validator.new.tap do |validator|
+        validator.validate(data, schema, project_file)
+
+        validator.warnings.each { |warning| logs << { warn: warning } }
+
+        validator.deprecations.each do |dep|
+          deprecations << { type: "#{CONFIG_NAME} #{dep[:option]}", msg: dep[:message] }
+        end
+      end
+
+      new(data, path, type, logs, deprecations)
     end
 
-    def initialize(raw_data, path, type = 'option', logs = [])
-      @path = Pathname.new(path).expand_path
+    # Builds the schema for bolt-project.yaml used by the validator.
+    #
+    def self.schema
+      {
+        type:        Hash,
+        properties:  Bolt::Config::BOLT_PROJECT_OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: Bolt::Config::OPTIONS
+      }
+    end
 
-      @project_file = @path + 'bolt-project.yaml'
+    def initialize(raw_data, path, type = 'option', logs = [], deprecations = [])
+      @path         = Pathname.new(path).expand_path
+      @project_file = @path + CONFIG_NAME
+      @logs         = logs
+      @deprecations = deprecations
 
-      @logs = logs
-      @deprecations = []
       if (@path + 'bolt.yaml').file? && project_file?
         msg = "Project-level configuration in bolt.yaml is deprecated if using bolt-project.yaml. "\
           "Transport config should be set in inventory.yaml, all other config should be set in "\
@@ -105,6 +129,7 @@ module Bolt
       @plans_path        = @path + 'plans'
       @managed_moduledir = @path + '.modules'
       @backup_dir        = @path + '.bolt-bak'
+      @cache_file        = @path + '.plugin_cache.json'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
@@ -173,6 +198,14 @@ module Bolt
       @data['plans']
     end
 
+    def plugin_cache
+      @data['plugin-cache']
+    end
+
+    def module_install
+      @data['module-install']
+    end
+
     def modules
       @modules ||= @data['modules']&.map do |mod|
         if mod.is_a?(String)
@@ -194,27 +227,13 @@ module Bolt
           raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
             "with a built-in Bolt module of the same name."
         end
-      else
+      elsif name.nil? &&
+            (File.directory?(plans_path) ||
+            File.directory?(@path + 'tasks') ||
+            File.directory?(@path + 'files'))
         message = "No project name is specified in bolt-project.yaml. Project-level content will not be available."
         @logs << { warn: message }
       end
-
-      %w[tasks plans].each do |conf|
-        unless @data.fetch(conf, []).is_a?(Array)
-          raise Bolt::ValidationError, "'#{conf}' in bolt-project.yaml must be an array"
-        end
-      end
-
-      if @data['modules']
-        unless @data['modules'].is_a?(Array)
-          raise Bolt::ValidationError, "'modules' in bolt-project.yaml must be an array"
-        end
-
-        @data['modules'].each do |spec|
-          next if spec.is_a?(Hash) || spec.is_a?(String)
-          raise Bolt::ValidationError, "Module specification #{spec.inspect} must be a hash or string"
-        end
-      end
     end
 
     def check_deprecated_file