openbolt 5.0.0.rc1

data/lib/bolt/plugin.rb

@@ -0,0 +1,380 @@
+# frozen_string_literal: true
+
+require_relative '../bolt/inventory'
+require_relative '../bolt/executor'
+require_relative '../bolt/module'
+require_relative '../bolt/pal'
+require_relative 'plugin/cache'
+require_relative 'plugin/puppetdb'
+
+module Bolt
+  class Plugin
+    KNOWN_HOOKS = %i[
+      puppet_library
+      resolve_reference
+      secret_encrypt
+      secret_decrypt
+      secret_createkeys
+      validate_resolve_reference
+    ].freeze
+
+    class PluginError < Bolt::Error
+      class ExecutionError < PluginError
+        def initialize(msg, plugin_name, location)
+          mess = "Error executing plugin #{plugin_name} from #{location}: #{msg}"
+          super(mess, 'bolt/plugin-error')
+        end
+      end
+
+      class Unknown < PluginError
+        def initialize(plugin_name)
+          super("Unknown plugin: '#{plugin_name}'", 'bolt/unknown-plugin')
+        end
+      end
+
+      class UnsupportedHook < PluginError
+        def initialize(plugin_name, hook)
+          super("Plugin #{plugin_name} does not support #{hook}", 'bolt/unsupported-hook')
+        end
+      end
+
+      class LoadingDisabled < PluginError
+        def initialize(plugin_name)
+          msg = "Cannot load plugin #{plugin_name}: plugin loading is disabled"
+          super(msg, 'bolt/plugin-loading-disabled', { 'plugin_name' => plugin_name })
+        end
+      end
+    end
+
+    class PluginContext
+      def initialize(config, pal, plugins)
+        @pal = pal
+        @config = config
+        @plugins = plugins
+      end
+
+      def serial_executor
+        @serial_executor ||= Bolt::Executor.new(1)
+      end
+      private :serial_executor
+
+      def empty_inventory
+        @empty_inventory ||= Bolt::Inventory.empty
+      end
+      private :empty_inventory
+
+      def with_a_compiler
+        # If we're already inside a pal compiler block use that compiler
+        # This may blow up if you try to load a task in catalog pal. Should we
+        # guard against that?
+        compiler = nil
+        if defined?(Puppet)
+          begin
+            compiler = Puppet.lookup(:pal_compiler)
+          rescue Puppet::Context::UndefinedBindingError; end # rubocop:disable Lint/SuppressedException
+        end
+
+        if compiler
+          yield compiler
+        else
+          @pal.in_bolt_compiler do |temp_compiler|
+            yield temp_compiler
+          end
+        end
+      end
+      private :with_a_compiler
+
+      def get_validated_task(task_name, params = nil)
+        with_a_compiler do |compiler|
+          tasksig = compiler.task_signature(task_name)
+
+          raise Bolt::Error.unknown_task(task_name) unless tasksig
+
+          Bolt::Task::Run.validate_params(tasksig, params) if params
+          Bolt::Task.from_task_signature(tasksig)
+        end
+      end
+
+      def validate_params(task_name, params)
+        with_a_compiler do |compiler|
+          tasksig = compiler.task_signature(task_name)
+
+          raise Bolt::Error.new("#{task_name} could not be found", 'bolt/plugin-error') unless tasksig
+
+          Bolt::Task::Run.validate_params(tasksig, params)
+        end
+        nil
+      end
+
+      # By passing `_` keys in params the caller can send metaparams directly to the task
+      # _catch_errors must be passed as an executor option not a param
+      def run_local_task(task, params, options)
+        # Make sure we're in a compiler to use the sensitive type
+        with_a_compiler do |_comp|
+          params = Bolt::Task::Run.wrap_sensitive(task, params)
+          Bolt::Task::Run.run_task(
+            task,
+            empty_inventory.get_targets('localhost'),
+            params,
+            options,
+            serial_executor
+          )
+        end
+      end
+
+      def boltdir
+        @config.project.path
+      end
+    end
+
+    RUBY_PLUGINS = %w[task prompt env_var puppetdb puppet_connect_data].freeze
+    BUILTIN_PLUGINS = %w[task terraform pkcs7 prompt vault aws_inventory puppetdb azure_inventory
+                         yaml env_var gcloud_inventory].freeze
+    DEFAULT_PLUGIN_HOOKS = { 'puppet_library' => { 'plugin' => 'puppet_agent', 'stop_service' => true } }.freeze
+
+    attr_reader :pal, :plugin_context
+    attr_writer :plugin_hooks
+
+    def initialize(config, pal, analytics = Bolt::Analytics::NoopClient.new, load_plugins: true)
+      @config = config
+      @analytics = analytics
+      @plugin_context = PluginContext.new(config, pal, self)
+      @plugins = {}
+      @pal = pal
+      @load_plugins = load_plugins
+      @unknown = Set.new
+      @resolution_stack = []
+      @unresolved_plugin_configs = config.plugins.dup
+      # The puppetdb plugin config comes from the puppetdb section, not from
+      # the plugins section
+      if @unresolved_plugin_configs.key?('puppetdb')
+        msg = "Configuration for the PuppetDB plugin must be in the 'puppetdb' config section, not 'plugins'"
+        raise Bolt::Error.new(msg, 'bolt/plugin-error')
+      end
+      @unresolved_plugin_configs['puppetdb'] = config.puppetdb.merge('default'   => config.default_puppetdb,
+                                                                     'instances' => config.puppetdb_instances)
+    end
+
+    # Returns a map of configured plugin hooks. Any unresolved plugin references
+    # are resolved.
+    #
+    # @return [Hash[String, Hash]]
+    #
+    def plugin_hooks
+      @plugin_hooks ||= DEFAULT_PLUGIN_HOOKS.merge(resolve_references(@config.plugin_hooks))
+    end
+
+    def modules
+      @modules ||= Bolt::Module.discover(@pal.full_modulepath, @config.project)
+    end
+
+    def add_plugin(plugin)
+      @plugins[plugin.name] = plugin
+    end
+
+    def add_ruby_plugin(plugin_name)
+      cls_name = Bolt::Util.snake_name_to_class_name(plugin_name)
+      filename = "bolt/plugin/#{plugin_name}"
+      require filename
+      cls = Kernel.const_get("Bolt::Plugin::#{cls_name}")
+      opts = {
+        context: @plugin_context,
+        config: config_for_plugin(plugin_name)
+      }
+
+      plugin = cls.new(**opts)
+      add_plugin(plugin)
+    end
+
+    def add_module_plugin(plugin_name)
+      opts = {
+        context: @plugin_context,
+        # Make sure that the plugin's config is validated _before_ the unknown-plugin
+        # and loading-disabled checks. This way, we can fail early on invalid plugin
+        # config instead of _after_ loading the modulepath (which can be expensive).
+        config: config_for_plugin(plugin_name)
+      }
+
+      mod = modules[plugin_name]
+
+      plugin = Bolt::Plugin::Module.load(mod, opts)
+      add_plugin(plugin)
+    end
+
+    def config_for_plugin(plugin_name)
+      return {} unless @unresolved_plugin_configs.include?(plugin_name)
+      if @resolution_stack.include?(plugin_name)
+        msg = "Configuration for plugin '#{plugin_name}' depends on the plugin itself"
+        raise PluginError.new(msg, 'bolt/plugin-error')
+      else
+        @resolution_stack.push(plugin_name)
+        config = resolve_references(@unresolved_plugin_configs[plugin_name])
+        @unresolved_plugin_configs.delete(plugin_name)
+        @resolution_stack.pop
+        config
+      end
+    end
+
+    def known_plugin?(plugin_name)
+      @plugins.include?(plugin_name) ||
+        RUBY_PLUGINS.include?(plugin_name) ||
+        (modules.include?(plugin_name) && modules[plugin_name].plugin?)
+    end
+
+    def get_hook(plugin_name, hook)
+      plugin = by_name(plugin_name)
+      raise PluginError::Unknown, plugin_name unless plugin
+      raise PluginError::UnsupportedHook.new(plugin_name, hook) unless plugin.hooks.include?(hook)
+      @analytics.report_bundled_content("Plugin #{hook}", plugin_name)
+
+      plugin.method(hook)
+    end
+
+    # Calling by_name or get_hook will load any module based plugin automatically
+    def by_name(plugin_name)
+      if known_plugin?(plugin_name)
+        if @plugins.include?(plugin_name)
+          @plugins[plugin_name]
+        elsif !@load_plugins
+          raise PluginError::LoadingDisabled, plugin_name
+        elsif RUBY_PLUGINS.include?(plugin_name)
+          add_ruby_plugin(plugin_name)
+        else
+          add_module_plugin(plugin_name)
+        end
+      end
+    end
+
+    # Loads all plugins and returns a map of plugin names to hooks.
+    #
+    def list_plugins
+      load_all_plugins
+
+      hooks = KNOWN_HOOKS.map { |hook| [hook, {}] }.to_h
+
+      @plugins.sort.each do |name, plugin|
+        # Don't show the Puppet Connect plugin for now.
+        next if name == 'puppet_connect_data'
+
+        case plugin
+        when Bolt::Plugin::Module
+          plugin.hook_map.each do |hook, spec|
+            next unless hooks.include?(hook)
+            hooks[hook][name] = spec['task'].description
+          end
+        else
+          plugin.hook_descriptions.each do |hook, description|
+            hooks[hook][name] = description
+          end
+        end
+      end
+
+      hooks
+    end
+
+    # Loads all plugins available to the project.
+    #
+    private def load_all_plugins
+      modules.each do |name, mod|
+        next unless mod.plugin?
+        by_name(name)
+      end
+
+      RUBY_PLUGINS.each { |name| by_name(name) }
+    end
+
+    def puppetdb_client
+      by_name('puppetdb').puppetdb_client
+    end
+
+    # Evaluate all _plugin references in a data structure. Leaves are
+    # evaluated and then their parents are evaluated with references replaced
+    # by their values. If the result of a reference contains more references,
+    # they are resolved again before continuing to ascend the tree. The final
+    # result will not contain any references.
+    def resolve_references(data)
+      Bolt::Util.postwalk_vals(data) do |value|
+        reference?(value) ? resolve_references(resolve_single_reference(value)) : value
+      end
+    rescue SystemStackError
+      raise Bolt::Error.new("Stack depth exceeded while recursively resolving references.",
+                            "bolt/recursive-reference-loop")
+    end
+
+    # Iteratively resolves "top-level" references until the result no longer
+    # has top-level references. A top-level reference is one which is not
+    # contained within another hash. It may be either the actual top-level
+    # result or arbitrarily nested within arrays. If parameters of the
+    # reference are themselves references, they will be looked. Any remaining
+    # references nested inside the result will *not* be evaluated once the
+    # top-level result is not a reference.  This is used to resolve the
+    # `targets` and `groups` keys which are allowed to be references or
+    # arrays of references, but which may return data with nested references
+    # that should be resolved lazily. The end result will either be a single
+    # hash or a flat array of hashes.
+    def resolve_top_level_references(data)
+      if data.is_a?(Array)
+        data.flat_map { |elem| resolve_top_level_references(elem) }
+      elsif reference?(data)
+        partially_resolved = data.transform_values do |v|
+          resolve_references(v)
+        end
+        fully_resolved = resolve_single_reference(partially_resolved)
+        # The top-level reference may have returned more references, so repeat the process
+        resolve_top_level_references(fully_resolved)
+      else
+        data
+      end
+    end
+
+    # Evaluates a single reference. The value returned may be another
+    # reference.
+    def resolve_single_reference(reference)
+      plugin_cache = if cache?(reference)
+                       cache = Bolt::Plugin::Cache.new(reference,
+                                                       @config.project.plugin_cache_file,
+                                                       @config.plugin_cache)
+                       entry = cache.read_and_clean_cache
+                       return entry unless entry.nil?
+
+                       cache
+                     end
+
+      plugin_name = reference['_plugin']
+      hook = get_hook(plugin_name, :resolve_reference)
+
+      begin
+        validate_proc = get_hook(plugin_name, :validate_resolve_reference)
+      rescue PluginError
+        validate_proc = proc { |*args| } # Nothing to do
+      end
+
+      validate_proc.call(reference)
+
+      result = begin
+        # Evaluate the plugin and then recursively evaluate any plugin returned by it.
+        hook.call(reference)
+      rescue StandardError => e
+        loc = "resolve_reference in #{plugin_name}"
+        raise PluginError::ExecutionError.new(e.message, plugin_name, loc)
+      end
+
+      plugin_cache.write_cache(result) if cache?(reference)
+
+      result
+    end
+    private :resolve_single_reference
+
+    private def cache?(reference)
+      reference.key?('_cache') || @config.plugin_cache.key?('ttl')
+    end
+
+    # Checks whether a given value is a _plugin reference
+    def reference?(input)
+      input.is_a?(Hash) && input.key?('_plugin')
+    end
+  end
+end
+
+# references PluginError
+require_relative 'plugin/module'

data/lib/bolt/project.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require_relative '../bolt/config'
+require_relative '../bolt/validator'
+require_relative '../bolt/pal'
+require_relative '../bolt/module'
+
+module Bolt
+  class Project
+    BOLTDIR_NAME = 'Boltdir'
+    CONFIG_NAME  = 'bolt-project.yaml'
+
+    attr_reader :path, :data, :inventory_file, :hiera_config,
+                :puppetfile, :rerunfile, :type, :resource_types, :project_file,
+                :downloads, :plans_path, :modulepath, :managed_moduledir,
+                :backup_dir, :plugin_cache_file, :plan_cache_file, :task_cache_file,
+                :manifests
+
+    def self.default_project
+      create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user')
+    # If homedir isn't defined use the system config path
+    rescue ArgumentError
+      create_project(Bolt::Config.system_path, 'system')
+    end
+
+    # Search recursively up the directory hierarchy for the Project. Look for a
+    # directory called Boltdir or a file called bolt-project.yaml (for a control
+    # repo type Project). Otherwise, repeat the check on each directory up the
+    # hierarchy, falling back to the default if we reach the root.
+    def self.find_boltdir(dir)
+      dir = Pathname.new(dir)
+
+      if (dir + BOLTDIR_NAME).directory?
+        create_project(dir + BOLTDIR_NAME, 'embedded')
+      elsif (dir + CONFIG_NAME).file?
+        create_project(dir, 'local')
+      elsif dir.root?
+        default_project
+      else
+        Bolt::Logger.debug(
+          "Did not detect Boltdir or bolt-project.yaml at '#{dir}'. This directory won't be loaded as a project."
+        )
+        find_boltdir(dir.parent)
+      end
+    end
+
+    def self.create_project(path, type = 'option')
+      fullpath = Pathname.new(path).expand_path
+
+      if type == 'user'
+        begin
+          # This is already expanded if the type is user
+          FileUtils.mkdir_p(path)
+        rescue StandardError
+          Bolt::Logger.warn(
+            "non_writeable_project",
+            "Could not create default project at #{path}. Continuing without a writeable project. "\
+            "Log and rerun files will not be written."
+          )
+        end
+      end
+
+      if type == 'option' && !File.directory?(path)
+        raise Bolt::Error.new("Could not find project at #{path}", "bolt/project-error")
+      end
+
+      if !Bolt::Util.windows? && type != 'environment' && fullpath.world_writable?
+        raise Bolt::Error.new(
+          "Project directory '#{fullpath}' is world-writable which poses a security risk. Set "\
+          "BOLT_PROJECT='#{fullpath}' to force the use of this project directory.",
+          "bolt/world-writable-error"
+        )
+      end
+
+      project_file = File.join(fullpath, CONFIG_NAME)
+      data         = Bolt::Util.read_optional_yaml_hash(File.expand_path(project_file), 'project')
+      default      = type =~ /user|system/ ? 'default ' : ''
+
+      if File.exist?(File.expand_path(project_file))
+        Bolt::Logger.info("Loaded #{default}project from '#{fullpath}'")
+      end
+
+      Bolt::Validator.new.tap do |validator|
+        validator.validate(data, schema, project_file)
+        validator.warnings.each { |warning| Bolt::Logger.warn(warning[:id], warning[:msg]) }
+        validator.deprecations.each { |dep| Bolt::Logger.deprecate(dep[:id], dep[:msg]) }
+      end
+
+      new(data, path, type)
+    end
+
+    # Builds the schema for bolt-project.yaml used by the validator.
+    #
+    def self.schema
+      {
+        type:        Hash,
+        properties:  Bolt::Config::PROJECT_OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: Bolt::Config::OPTIONS
+      }
+    end
+
+    def initialize(data, path, type = 'option')
+      @type              = type
+      @path              = Pathname.new(path).expand_path
+      @project_file      = @path + CONFIG_NAME
+      @inventory_file    = @path + 'inventory.yaml'
+      @hiera_config      = @path + 'hiera.yaml'
+      @puppetfile        = @path + 'Puppetfile'
+      @rerunfile         = @path + '.rerun.json'
+      @resource_types    = @path + '.resource_types'
+      @downloads         = @path + 'downloads'
+      @plans_path        = @path + 'plans'
+      @managed_moduledir = @path + '.modules'
+      @backup_dir        = @path + '.bolt-bak'
+      @plugin_cache_file = @path + '.plugin_cache.json'
+      @plan_cache_file   = @path + '.plan_cache.json'
+      @task_cache_file   = @path + '.task_cache.json'
+      @modulepath        = [(@path + 'modules').to_s]
+      @manifests         = @path + 'manifests'
+
+      if (tc = Bolt::Config::INVENTORY_OPTIONS.keys & data.keys).any?
+        Bolt::Logger.warn(
+          "project_transport_config",
+          "Transport configuration isn't supported in bolt-project.yaml. Ignoring keys #{tc}."
+        )
+      end
+
+      @data = data.slice(*Bolt::Config::PROJECT_OPTIONS)
+
+      if @data['rerunfile']
+        @rerunfile = File.expand_path(@data['rerunfile'], @path)
+      end
+
+      validate if project_file?
+    end
+
+    def to_s
+      @path.to_s
+    end
+
+    # This API is used to prepend the project as a module to Puppet's internal
+    # module_references list. CHANGE AT YOUR OWN RISK
+    def to_h
+      { path: @path.to_s,
+        name: name,
+        load_as_module?: load_as_module? }
+    end
+
+    def eql?(other)
+      path == other.path
+    end
+    alias == eql?
+
+    def project_file?
+      @project_file.file?
+    end
+
+    def load_as_module?
+      !name.nil?
+    end
+
+    def name
+      @data['name']
+    end
+
+    def tasks
+      @data['tasks']
+    end
+
+    def plans
+      @data['plans']
+    end
+
+    def plugin_cache
+      @data['plugin-cache']
+    end
+
+    def module_install
+      @data['module-install']
+    end
+
+    def disable_warnings
+      @data['disable-warnings'] || []
+    end
+
+    def modules
+      mod_data = @data['modules'] || []
+      @modules ||= mod_data.map do |mod|
+        if mod.is_a?(String)
+          { 'name' => mod }
+        else
+          mod
+        end
+      end
+    end
+
+    def validate
+      if name
+        if name !~ Bolt::Module::MODULE_NAME_REGEX
+          raise Bolt::ValidationError, <<~ERROR_STRING
+          Invalid project name '#{name}' in bolt-project.yaml; project name must begin with a lowercase letter
+          and can include lowercase letters, numbers, and underscores.
+          ERROR_STRING
+        elsif Dir.children(Bolt::Config::Modulepath::BOLTLIB_PATH).include?(name)
+          raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
+            "with a built-in Bolt module of the same name."
+        end
+      elsif name.nil? &&
+            (File.directory?(plans_path) ||
+            File.directory?(@path + 'tasks') ||
+            File.directory?(@path + 'files'))
+        message = "No project name is specified in bolt-project.yaml. Project-level content will not be available."
+
+        Bolt::Logger.warn("missing_project_name", message)
+      end
+    end
+  end
+end

data/lib/bolt/project_manager/config_migrator.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require_relative '../../bolt/project_manager/migrator'
+
+module Bolt
+  class ProjectManager
+    class ConfigMigrator < Migrator
+      def migrate(config_file, project_file, inventory_file, backup_dir)
+        bolt_yaml_to_bolt_project(config_file, project_file, inventory_file, backup_dir) &&
+          update_options(project_file)
+      end
+
+      private def bolt_yaml_to_bolt_project(config_file, project_file, inventory_file, backup_dir)
+        if File.exist?(project_file)
+          return true
+        end
+
+        unless File.exist?(config_file)
+          return true
+        end
+
+        @outputter.print_message "Migrating project configuration\n\n"
+
+        config_data = Bolt::Util.read_optional_yaml_hash(config_file, 'config')
+        transport_data, project_data = config_data.partition do |k, _|
+          Bolt::Config::INVENTORY_OPTIONS.keys.include?(k)
+        end.map(&:to_h)
+
+        if transport_data.any?
+          if File.exist?(inventory_file)
+            inventory_data = Bolt::Util.read_yaml_hash(inventory_file, 'inventory')
+            merged = Bolt::Util.deep_merge(transport_data, inventory_data['config'] || {})
+            inventory_data['config'] = merged
+            backup_file(inventory_file, backup_dir)
+          else
+            FileUtils.touch(inventory_file)
+            inventory_data = { 'config' => transport_data }
+          end
+
+          backup_file(config_file, backup_dir)
+
+          begin
+            @outputter.print_action_step(
+              "Moving transportation configuration options '#{transport_data.keys.join(', ')}' "\
+              "from bolt.yaml to inventory.yaml"
+            )
+
+            File.write(inventory_file, inventory_data.to_yaml)
+            File.write(config_file, project_data.to_yaml)
+          rescue StandardError => e
+            raise Bolt::FileError.new("#{e.message}; unable to write inventory.", inventory_file)
+          end
+        end
+
+        @outputter.print_action_step("Renaming bolt.yaml to bolt-project.yaml")
+        FileUtils.mv(config_file, project_file)
+
+        command = Bolt::Util.powershell? ? 'Get-Help about_bolt_project' : 'bolt guide project'
+        @outputter.print_action_step(
+          "Successfully migrated config. Please add a 'name' key to bolt-project.yaml "\
+          "to use project-level tasks and plans. Learn more about projects by running "\
+          "'#{command}'."
+        )
+
+        true
+      end
+
+      private def update_options(project_file)
+        return true unless File.exist?(project_file)
+
+        @outputter.print_message("Updating project configuration options\n\n")
+        data     = Bolt::Util.read_yaml_hash(project_file, 'config')
+        modified = false
+
+        # Keys to update. The first element is the old key, while the second is
+        # the key update it to.
+        to_update = [
+          %w[apply_settings apply-settings],
+          %w[puppetfile module-install],
+          %w[plugin_hooks plugin-hooks]
+        ]
+
+        to_update.each do |old, new|
+          next unless data.key?(old)
+
+          if data.key?(new)
+            @outputter.print_action_step("Removing deprecated option '#{old}'")
+            data.delete(old)
+          else
+            @outputter.print_action_step("Updating deprecated option '#{old}' to '#{new}'")
+            data[new] = data.delete(old)
+          end
+
+          modified = true
+        end
+
+        if modified
+          begin
+            File.write(project_file, data.to_yaml)
+          rescue StandardError => e
+            raise Bolt::FileError.new("#{e.message}; unable to write config.", project_file)
+          end
+
+          @outputter.print_action_step("Successfully updated project configuration #{project_file}")
+        else
+          @outputter.print_action_step("Project configuration is up to date, nothing to do.")
+        end
+
+        true
+      end
+    end
+  end
+end