bolt 2.37.0 → 2.44.0

data/lib/bolt/pal.rb

@@ -153,8 +153,10 @@ module Bolt
         Dir.children(path).select { |name| Puppet::Module.is_module_directory?(name, path) }
       end
       if modules.include?(project.name)
-        Bolt::Logger.warn_once("project shadows module",
-                               "The project '#{project.name}' shadows an existing module of the same name")
+        Bolt::Logger.warn_once(
+          "project_shadows_module",
+          "The project '#{project.name}' shadows an existing module of the same name"
+        )
       end
     end
 
@@ -357,19 +359,52 @@ module Bolt
       Bolt::Task.from_task_signature(task)
     end
 
+    def list_plans_with_cache(filter_content: false)
+      # Don't filter content yet, so that if users update their plan filters
+      # we don't need to refresh the cache
+      plan_names = list_plans(filter_content: false).map(&:first)
+      plan_cache = if @project
+                     Bolt::Util.read_optional_json_file(@project.plan_cache_file, 'Plan cache file')
+                   else
+                     {}
+                   end
+      updated = false
+
+      plan_list = plan_names.each_with_object([]) do |plan_name, list|
+        info = plan_cache[plan_name] || get_plan_info(plan_name, with_mtime: true)
+
+        # If the plan is a 'local' plan (in the project itself, or the
+        # modules/ directory) then verify it hasn't been updated since we
+        # cached it. If it has been updated, refresh the cache and use the
+        # new data.
+        if info['file'] &&
+           (File.mtime(info.dig('file', 'path')) <=> info.dig('file', 'mtime')) != 0
+          info = get_plan_info(plan_name, with_mtime: true)
+          updated = true
+          plan_cache[plan_name] = info
+        end
+
+        list << [plan_name] unless info['private']
+      end
+
+      File.write(@project.plan_cache_file, plan_cache.to_json) if updated
+
+      filter_content ? filter_content(plan_list, @project&.plans) : plan_list
+    end
+
     def list_plans(filter_content: false)
       in_bolt_compiler do |compiler|
         errors = []
         plans = compiler.list_plans(nil, errors).map { |plan| [plan.name] }.sort
         errors.each do |error|
-          @logger.warn(error.details['original_error'])
+          Bolt::Logger.warn("plan_load_error", error.details['original_error'])
         end
 
         filter_content ? filter_content(plans, @project&.plans) : plans
       end
     end
 
-    def get_plan_info(plan_name)
+    def get_plan_info(plan_name, with_mtime: false)
       plan_sig = in_bolt_compiler do |compiler|
         compiler.plan_signature(plan_name)
       end
@@ -412,16 +447,28 @@ module Bolt
             params[name]['default_value'] = defaults[name] if defaults.key?(name)
             params[name]['description'] = param.text unless param.text.empty?
           else
-            @logger.warn("The documented parameter '#{name}' does not exist in plan signature")
+            Bolt::Logger.warn(
+              "missing_plan_parameter",
+              "The documented parameter '#{name}' does not exist in plan signature"
+            )
           end
         end
 
-        {
-          'name' => plan_name,
+        privie = plan.tag(:private)&.text
+        unless privie.nil? || %w[true false].include?(privie.downcase)
+          msg = "Plan #{plan_name} key 'private' must be a boolean, received: #{privie}"
+          raise Bolt::Error.new(msg, 'bolt/invalid-plan')
+        end
+
+        pp_info = {
+          'name'        => plan_name,
           'description' => description,
-          'parameters' => parameters,
-          'module' => mod
+          'parameters'  => parameters,
+          'module'      => mod
         }
+        pp_info.merge!({ 'private' => privie&.downcase == 'true' }) unless privie.nil?
+        pp_info.merge!(get_plan_mtime(plan.file)) if with_mtime
+        pp_info
 
       # If it's a YAML plan, fall back to limited data
       else
@@ -444,12 +491,32 @@ module Bolt
           params[name]['default_value'] = param.value unless param.value.nil?
           params[name]['description'] = param.description if param.description
         end
-        {
-          'name' => plan_name,
+
+        yaml_info = {
+          'name'        => plan_name,
           'description' => plan.description,
-          'parameters' => parameters,
-          'module' => mod
+          'parameters'  => parameters,
+          'module'      => mod
         }
+        yaml_info.merge!({ 'private' => plan.private }) unless plan.private.nil?
+        yaml_info.merge!(get_plan_mtime(yaml_path)) if with_mtime
+        yaml_info
+      end
+    end
+
+    def get_plan_mtime(path)
+      # If the plan is from the project modules/ directory, or is in the
+      # project itself, include the last mtime of the file so we can compare
+      # if the plan has been updated since it was cached.
+      if @project &&
+         File.exist?(path) &&
+         (path.include?(File.join(@project.path, 'modules')) ||
+          path.include?(@project.plans_path.to_s))
+
+        { 'file' => { 'mtime' => File.mtime(path),
+                      'path' => path } }
+      else
+        {}
       end
     end
 
@@ -490,16 +557,28 @@ module Bolt
       end
     end
 
-    def generate_types
+    def generate_types(cache: false)
       require 'puppet/face/generate'
       in_bolt_compiler do
         generator = Puppet::Generate::Type
         inputs = generator.find_inputs(:pcore)
         FileUtils.mkdir_p(@resource_types)
+        cache_plan_info if @project && cache
         generator.generate(inputs, @resource_types, true)
       end
     end
 
+    def cache_plan_info
+      # plan_name is an array here
+      plans_info = list_plans(filter_content: false).map do |plan_name,|
+        data = get_plan_info(plan_name, with_mtime: true)
+        { plan_name => data }
+      end.reduce({}, :merge)
+
+      FileUtils.touch(@project.plan_cache_file)
+      File.write(@project.plan_cache_file, plans_info.to_json)
+    end
+
     def run_task(task_name, targets, params, executor, inventory, description = nil)
       in_task_compiler(executor, inventory) do |compiler|
         params = params.merge('_bolt_api_call' => true, '_catch_errors' => true)

data/lib/bolt/pal/yaml_plan.rb

@@ -6,10 +6,10 @@ require 'bolt/pal/yaml_plan/step'
 module Bolt
   class PAL
     class YamlPlan
-      PLAN_KEYS = Set['parameters', 'steps', 'return', 'version', 'description']
+      PLAN_KEYS = Set['parameters', 'private', 'steps', 'return', 'version', 'description']
       VAR_NAME_PATTERN = /\A[a-z_][a-z0-9_]*\z/.freeze
 
-      attr_reader :name, :parameters, :steps, :return, :description
+      attr_reader :name, :parameters, :private, :steps, :return, :description
 
       def initialize(name, plan)
         # Top-level plan keys aren't allowed to be Puppet code, so force them
@@ -30,6 +30,12 @@ module Bolt
           Parameter.new(param, definition)
         end.freeze
 
+        @private = plan['private']
+        unless @private.nil? || @private.is_a?(TrueClass) || @private.is_a?(FalseClass)
+          msg = "Plan #{@name} key 'private' must be a boolean, received: #{@private.inspect}"
+          raise Bolt::Error.new(msg, "bolt/invalid-plan")
+        end
+
         # Validate top level plan keys
         top_level_keys = plan.keys.to_set
         unless PLAN_KEYS.superset?(top_level_keys)

data/lib/bolt/pal/yaml_plan/evaluator.rb

@@ -157,13 +157,13 @@ module Bolt
           if plan.steps.any? { |step| step.body.key?('target') }
             msg = "The 'target' parameter for YAML plan steps is deprecated and will be removed "\
                   "in a future version of Bolt. Use the 'targets' parameter instead."
-            Bolt::Logger.deprecation_warning("Using 'target' parameter for YAML plan steps, not 'targets'", msg)
+            Bolt::Logger.deprecate("yaml_plan_target", msg)
           end
 
           if plan.steps.any? { |step| step.body.key?('source') }
             msg = "The 'source' parameter for YAML plan upload steps is deprecated and will be removed "\
                   "in a future version of Bolt. Use the 'upload' parameter instead."
-            Bolt::Logger.deprecation_warning("Using 'source' parameter for YAML upload steps, not 'upload'", msg)
+            Bolt::Logger.deprecate("yaml_plan_source", msg)
           end
 
           plan_result = closure_scope.with_local_scope(args_hash) do |scope|

data/lib/bolt/pal/yaml_plan/transpiler.rb

@@ -30,6 +30,7 @@ module Bolt
           plan_string = String.new('')
           plan_string << "# #{plan_object.description}\n" if plan_object.description
           plan_string << "# WARNING: This is an autogenerated plan. It may not behave as expected.\n"
+          plan_string << "# @private #{plan_object.private}\n" unless plan_object.private.nil?
           plan_string << "#{param_descriptions}\n" unless param_descriptions.empty?
 
           plan_string << "plan #{plan_object.name}("
@@ -64,7 +65,11 @@ module Bolt
             raise Bolt::FileError.new(msg, @plan_path)
           end
 
-          Bolt::PAL::YamlPlan::Loader.from_string(@modulename, file_contents, @plan_path)
+          begin
+            Bolt::PAL::YamlPlan::Loader.from_string(@modulename, file_contents, @plan_path)
+          rescue Puppet::PreformattedError, StandardError => e
+            raise PALError.from_preformatted_error(e)
+          end
         end
 
         def validate_path

data/lib/bolt/plugin.rb

@@ -139,7 +139,7 @@ module Bolt
       plugins
     end
 
-    RUBY_PLUGINS = %w[task prompt env_var puppetdb].freeze
+    RUBY_PLUGINS = %w[task prompt env_var puppetdb puppet_connect_data].freeze
     BUILTIN_PLUGINS = %w[task terraform pkcs7 prompt vault aws_inventory puppetdb azure_inventory
                          yaml env_var gcloud_inventory].freeze
     DEFAULT_PLUGIN_HOOKS = { 'puppet_library' => { 'plugin' => 'puppet_agent', 'stop_service' => true } }.freeze
@@ -170,7 +170,7 @@ module Bolt
     end
 
     def modules
-      @modules ||= Bolt::Module.discover(@pal.full_modulepath)
+      @modules ||= Bolt::Module.discover(@pal.full_modulepath, @config.project)
     end
 
     def add_plugin(plugin)
@@ -297,7 +297,7 @@ module Bolt
     def resolve_single_reference(reference)
       plugin_cache = if cache?(reference)
                        cache = Bolt::Plugin::Cache.new(reference,
-                                                       @config.project.cache_file,
+                                                       @config.project.plugin_cache_file,
                                                        @config.plugin_cache)
                        entry = cache.read_and_clean_cache
                        return entry unless entry.nil?

data/lib/bolt/plugin/cache.rb

@@ -7,11 +7,11 @@ require 'bolt/util'
 module Bolt
   class Plugin
     class Cache
-      attr_reader :reference, :cache_file, :default_config, :id
+      attr_reader :reference, :plugin_cache_file, :default_config, :id
 
-      def initialize(reference, cache_file, default_config)
+      def initialize(reference, plugin_cache_file, default_config)
         @reference = reference
-        @cache_file = cache_file
+        @plugin_cache_file = plugin_cache_file
         @default_config = default_config
       end
 
@@ -21,7 +21,7 @@ module Bolt
 
         # Luckily we don't need to use a serious hash algorithm
         require 'digest/bubblebabble'
-        r = reference.select { |k, _| k == '_cache' }.sort.to_s
+        r = reference.reject { |k, _| k == '_cache' }.sort.to_s
         @id = Digest::SHA2.bubblebabble(r)[0..20]
 
         unmodified = true
@@ -32,21 +32,21 @@ module Bolt
           unmodified = false if expired
           expired
         end
-        File.write(cache_file, cache.to_json) unless cache.empty? || unmodified
+        File.write(plugin_cache_file, cache.to_json) unless cache.empty? || unmodified
 
         cache.dig(id, 'result')
       end
 
       private def cache
-        @cache ||= Bolt::Util.read_optional_json_file(@cache_file, 'cache')
+        @cache ||= Bolt::Util.read_optional_json_file(@plugin_cache_file, 'cache')
       end
 
       def write_cache(result)
         cache.merge!({ id => { 'result' => result,
                                'mtime' => Time.now,
                                'ttl' => ttl } })
-        FileUtils.touch(cache_file)
-        File.write(cache_file, cache.to_json)
+        FileUtils.touch(plugin_cache_file)
+        File.write(plugin_cache_file, cache.to_json)
       end
 
       def validate

data/lib/bolt/plugin/module.rb

@@ -187,7 +187,7 @@ module Bolt
         if params.key?('private-key') || params.key?('public-key')
           message = "pkcs7 keys 'private-key' and 'public-key' have been deprecated and will be "\
                     "removed in a future version of Bolt; use 'private_key' and 'public_key' instead."
-          Bolt::Logger.deprecation_warning('PKCS7 keys using hyphens, not underscores', message)
+          Bolt::Logger.deprecate("pkcs7_params", message)
         end
 
         params['private_key'] = params.delete('private-key') if params.key?('private-key')

data/lib/bolt/plugin/puppet_connect_data.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Bolt
+  class Plugin
+    class PuppetConnectData
+      def initialize(context:, **_opts)
+        puppet_connect_data_yaml_path = File.join(context.boltdir, 'puppet_connect_data.yaml')
+        @data = Bolt::Util.read_optional_yaml_hash(
+          puppet_connect_data_yaml_path,
+          'puppet_connect_data.yaml'
+        )
+      end
+
+      def name
+        'puppet_connect_data'
+      end
+
+      def hooks
+        %i[resolve_reference validate_resolve_reference]
+      end
+
+      def resolve_reference(opts)
+        key = opts['key']
+        @data[key]
+      end
+
+      def validate_resolve_reference(opts)
+        unless opts['key']
+          raise Bolt::ValidationError,
+                "puppet_connect_data plugin requires that 'key' be specified"
+        end
+      end
+    end
+  end
+end

data/lib/bolt/plugin/puppetdb.rb

@@ -12,7 +12,7 @@ module Bolt
       end
 
       TEMPLATE_OPTS = %w[alias config facts features name uri vars].freeze
-      PLUGIN_OPTS = %w[_plugin query target_mapping].freeze
+      PLUGIN_OPTS = %w[_plugin _cache query target_mapping].freeze
 
       attr_reader :puppetdb_client
 
@@ -31,7 +31,7 @@ module Bolt
       end
 
       def warn_missing_fact(certname, fact)
-        @logger.warn("Could not find fact #{fact} for node #{certname}")
+        Bolt::Logger.warn("puppetdb_missing_fact", "Could not find fact #{fact} for node #{certname}")
       end
 
       def fact_path(raw_fact)

data/lib/bolt/project.rb

@@ -2,7 +2,7 @@
 
 require 'pathname'
 require 'bolt/config'
-require 'bolt/config/validator'
+require 'bolt/validator'
 require 'bolt/pal'
 require 'bolt/module'
 
@@ -12,46 +12,49 @@ module Bolt
     CONFIG_NAME  = 'bolt-project.yaml'
 
     attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
-                :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
-                :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
-                :backup_dir, :cache_file
+                :puppetfile, :rerunfile, :type, :resource_types, :project_file,
+                :downloads, :plans_path, :modulepath, :managed_moduledir,
+                :backup_dir, :plugin_cache_file, :plan_cache_file
 
-    def self.default_project(logs = [])
-      create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
+    def self.default_project
+      create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user')
     # If homedir isn't defined use the system config path
     rescue ArgumentError
-      create_project(Bolt::Config.system_path, 'system', logs)
+      create_project(Bolt::Config.system_path, 'system')
     end
 
     # Search recursively up the directory hierarchy for the Project. Look for a
     # directory called Boltdir or a file called bolt.yaml (for a control repo
     # type Project). Otherwise, repeat the check on each directory up the
     # hierarchy, falling back to the default if we reach the root.
-    def self.find_boltdir(dir, logs = [], deprecations = [])
+    def self.find_boltdir(dir)
       dir = Pathname.new(dir)
 
       if (dir + BOLTDIR_NAME).directory?
-        create_project(dir + BOLTDIR_NAME, 'embedded', logs)
+        create_project(dir + BOLTDIR_NAME, 'embedded')
       elsif (dir + 'bolt.yaml').file?
         command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
-        msg = "Configuration file #{dir + 'bolt.yaml'} is deprecated and will be "\
+        Bolt::Logger.deprecate(
+          "bolt_yaml",
+          "Configuration file #{dir + 'bolt.yaml'} is deprecated and will be "\
           "removed in Bolt 3.0.\nUpdate your Bolt project to the latest Bolt practices "\
-          "using #{command}"
-        deprecations << { type: "Project level bolt.yaml",
-                          msg: msg }
-        create_project(dir, 'local', logs, deprecations)
+          "using #{command}."
+        )
+        create_project(dir, 'local')
       elsif (dir + CONFIG_NAME).file?
-        create_project(dir, 'local', logs)
+        create_project(dir, 'local')
       elsif dir.root?
-        default_project(logs)
+        default_project
       else
-        logs << { debug: "Did not detect Boltdir, bolt.yaml, or bolt-project.yaml at '#{dir}'. "\
-                  "This directory won't be loaded as a project." }
-        find_boltdir(dir.parent, logs, deprecations)
+        Bolt::Logger.debug(
+          "Did not detect Boltdir, bolt.yaml, or bolt-project.yaml at '#{dir}'. "\
+          "This directory won't be loaded as a project."
+        )
+        find_boltdir(dir.parent)
       end
     end
 
-    def self.create_project(path, type = 'option', logs = [], deprecations = [])
+    def self.create_project(path, type = 'option')
       fullpath = Pathname.new(path).expand_path
 
       if type == 'user'
@@ -59,8 +62,11 @@ module Bolt
           # This is already expanded if the type is user
           FileUtils.mkdir_p(path)
         rescue StandardError
-          logs << { warn: "Could not create default project at #{path}. Continuing without a writeable project. "\
-                    "Log and rerun files will not be written." }
+          Bolt::Logger.warn(
+            "non_writeable_project",
+            "Could not create default project at #{path}. Continuing without a writeable project. "\
+            "Log and rerun files will not be written."
+          )
         end
       end
 
@@ -81,36 +87,40 @@ module Bolt
       default      = type =~ /user|system/ ? 'default ' : ''
       exist        = File.exist?(File.expand_path(project_file))
 
-      logs << { info: "Loaded #{default}project from '#{fullpath}'" } if exist
-
-      # Validate the config against the schema. This will raise a single error
-      # with all validation errors.
-      schema = Bolt::Config::OPTIONS.slice(*Bolt::Config::BOLT_PROJECT_OPTIONS)
+      if exist
+        Bolt::Logger.info("Loaded #{default}project from '#{fullpath}'")
+      end
 
-      Bolt::Config::Validator.new.tap do |validator|
+      Bolt::Validator.new.tap do |validator|
         validator.validate(data, schema, project_file)
-
-        validator.warnings.each { |warning| logs << { warn: warning } }
-
-        validator.deprecations.each do |dep|
-          deprecations << { type: "#{CONFIG_NAME} #{dep[:option]}", msg: dep[:message] }
-        end
+        validator.warnings.each { |warning| Bolt::Logger.warn(warning[:id], warning[:msg]) }
+        validator.deprecations.each { |dep| Bolt::Logger.deprecate(dep[:id], dep[:msg]) }
       end
 
-      new(data, path, type, logs, deprecations)
+      new(data, path, type)
+    end
+
+    # Builds the schema for bolt-project.yaml used by the validator.
+    #
+    def self.schema
+      {
+        type:        Hash,
+        properties:  Bolt::Config::BOLT_PROJECT_OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: Bolt::Config::OPTIONS
+      }
     end
 
-    def initialize(raw_data, path, type = 'option', logs = [], deprecations = [])
+    def initialize(raw_data, path, type = 'option')
       @path         = Pathname.new(path).expand_path
       @project_file = @path + CONFIG_NAME
-      @logs         = logs
-      @deprecations = deprecations
 
       if (@path + 'bolt.yaml').file? && project_file?
-        msg = "Project-level configuration in bolt.yaml is deprecated if using bolt-project.yaml. "\
+        Bolt::Logger.deprecate(
+          "bolt_yaml",
+          "Project-level configuration in bolt.yaml is deprecated if using bolt-project.yaml. "\
           "Transport config should be set in inventory.yaml, all other config should be set in "\
           "bolt-project.yaml."
-        @deprecations << { type: 'Using bolt.yaml for project configuration', msg: msg }
+        )
       end
 
       @inventory_file    = @path + 'inventory.yaml'
@@ -123,12 +133,14 @@ module Bolt
       @plans_path        = @path + 'plans'
       @managed_moduledir = @path + '.modules'
       @backup_dir        = @path + '.bolt-bak'
-      @cache_file        = @path + '.plugin_cache.json'
+      @plugin_cache_file = @path + '.plugin_cache.json'
+      @plan_cache_file   = @path + '.plan_cache.json'
 
-      tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
-      if tc.any?
-        msg = "Transport configuration isn't supported in bolt-project.yaml. Ignoring keys #{tc}"
-        @logs << { warn: msg }
+      if (tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys).any?
+        Bolt::Logger.warn(
+          "project_transport_config",
+          "Transport configuration isn't supported in bolt-project.yaml. Ignoring keys #{tc}."
+        )
       end
 
       @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_OPTIONS.include?(k) }
@@ -145,8 +157,10 @@ module Bolt
       # and replaced with .project_file in lib/bolt/config.rb
       @config_file = if (Bolt::Config::BOLT_OPTIONS & @data.keys).any?
                        if (@path + 'bolt.yaml').file?
-                         msg = "bolt-project.yaml contains valid config keys, bolt.yaml will be ignored"
-                         @logs << { warn: msg }
+                         Bolt::Logger.warn(
+                           "project_config_conflict",
+                           "bolt-project.yaml contains valid config keys, bolt.yaml will be ignored"
+                         )
                        end
                        @project_file
                      else
@@ -196,6 +210,14 @@ module Bolt
       @data['plugin-cache']
     end
 
+    def module_install
+      @data['module-install']
+    end
+
+    def disable_warnings
+      @data['disable-warnings'] || []
+    end
+
     def modules
       @modules ||= @data['modules']&.map do |mod|
         if mod.is_a?(String)
@@ -217,16 +239,20 @@ module Bolt
           raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
             "with a built-in Bolt module of the same name."
         end
-      else
+      elsif name.nil? &&
+            (File.directory?(plans_path) ||
+            File.directory?(@path + 'tasks') ||
+            File.directory?(@path + 'files'))
         message = "No project name is specified in bolt-project.yaml. Project-level content will not be available."
-        @logs << { warn: message }
+
+        Bolt::Logger.warn("missing_project_name", message)
       end
     end
 
     def check_deprecated_file
       if (@path + 'project.yaml').file?
         msg = "Project configuration file 'project.yaml' is deprecated; use 'bolt-project.yaml' instead."
-        Bolt::Logger.deprecation_warning('Using project.yaml instead of bolt-project.yaml', msg)
+        Bolt::Logger.warn("project_yaml", msg)
       end
     end
   end