bolt 2.29.0 → 2.33.2

data/lib/bolt/outputter/logger.rb

@@ -13,9 +13,9 @@ module Bolt
       def handle_event(event)
         case event[:type]
         when :step_start
-          log_step_start(event)
+          log_step_start(**event)
         when :step_finish
-          log_step_finish(event)
+          log_step_finish(**event)
         when :plan_start
           log_plan_start(event)
         when :plan_finish

data/lib/bolt/pal.rb

@@ -5,40 +5,36 @@ require 'bolt/executor'
 require 'bolt/error'
 require 'bolt/plan_result'
 require 'bolt/util'
+require 'bolt/config/modulepath'
 require 'etc'
 
 module Bolt
   class PAL
-    BOLTLIB_PATH = File.expand_path('../../bolt-modules', __dir__)
-    MODULES_PATH = File.expand_path('../../modules', __dir__)
-
     # PALError is used to convert errors from executing puppet code into
     # Bolt::Errors
     class PALError < Bolt::Error
       def self.from_preformatted_error(err)
-        if err.cause&.is_a? Bolt::Error
-          err.cause
-        else
-          from_error(err)
-        end
+        error = if err.cause.is_a? Bolt::Error
+                  err.cause
+                else
+                  from_error(err)
+                end
+
+        # Provide the location of an error if it came from a plan
+        details = {}
+        details[:file]   = err.file if defined?(err.file)
+        details[:line]   = err.line if defined?(err.line)
+        details[:column] = err.pos if defined?(err.pos)
+
+        error.add_filelineno(details)
+        error
       end
 
       # Generate a Bolt::Pal::PALError for non-bolt errors
       def self.from_error(err)
         # Use the original error message if available
         message = err.cause ? err.cause.message : err.message
-
-        # Provide the location of an error if it came from a plan
-        details = if defined?(err.file) && err.file
-                    { file:   err.file,
-                      line:   err.line,
-                      column: err.pos }.compact
-                  else
-                    {}
-                  end
-
-        e = new(message, details)
-
+        e = new(message)
         e.set_backtrace(err.backtrace)
         e
       end
@@ -48,16 +44,16 @@ module Bolt
       end
     end
 
-    attr_reader :modulepath, :user_modulepath
-
     def initialize(modulepath, hiera_config, resource_types, max_compiles = Etc.nprocessors,
                    trusted_external = nil, apply_settings = {}, project = nil)
+      unless modulepath.is_a?(Bolt::Config::Modulepath)
+        msg = "Type error in PAL: modulepath must be a Bolt::Config::Modulepath"
+        raise Bolt::Error.new(msg, "bolt/execution-error")
+      end
       # Nothing works without initialized this global state. Reinitializing
       # is safe and in practice only happens in tests
       self.class.load_puppet
-
-      @user_modulepath = modulepath
-      @modulepath = [BOLTLIB_PATH, *modulepath, MODULES_PATH]
+      @modulepath = modulepath
       @hiera_config = hiera_config
       @trusted_external = trusted_external
       @apply_settings = apply_settings
@@ -66,13 +62,21 @@ module Bolt
       @project = project
 
       @logger = Bolt::Logger.logger(self)
-      if modulepath && !modulepath.empty?
-        @logger.debug("Loading modules from #{@modulepath.join(File::PATH_SEPARATOR)}")
+      unless user_modulepath.empty?
+        @logger.debug("Loading modules from #{full_modulepath.join(File::PATH_SEPARATOR)}")
       end
 
       @loaded = false
     end
 
+    def full_modulepath
+      @modulepath.full_modulepath
+    end
+
+    def user_modulepath
+      @modulepath.user_modulepath
+    end
+
     # Puppet logging is global so this is class method to avoid confusion
     def self.configure_logging
       Puppet::Util::Log.destinations.clear
@@ -160,7 +164,7 @@ module Bolt
     def in_bolt_compiler
       # TODO: If we always call this inside a bolt_executor we can remove this here
       setup
-      r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
+      r = Puppet::Pal.in_tmp_environment('bolt', modulepath: full_modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
         Puppet.override(bolt_project: @project,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
@@ -215,11 +219,11 @@ module Bolt
         apply_executor: applicator || Applicator.new(
           inventory,
           executor,
-          @modulepath,
+          full_modulepath,
           # Skip syncing built-in plugins, since we vendor some Puppet 6
           # versions of "core" types, which are already present on the agent,
           # but may cause issues on Puppet 5 agents.
-          @user_modulepath,
+          user_modulepath,
           @project,
           pdb_client,
           @hiera_config,
@@ -253,19 +257,24 @@ module Bolt
 
     # TODO: PUP-8553 should replace this
     def with_puppet_settings
-      Dir.mktmpdir('bolt') do |dir|
-        cli = []
-        Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
-          cli << "--#{setting}" << dir
-        end
-        Puppet.settings.send(:clear_everything_for_tests)
-        Puppet.initialize_settings(cli)
-        Puppet::GettextConfig.create_default_text_domain
-        Puppet[:trusted_external_command] = @trusted_external
-        Puppet.settings[:hiera_config] = @hiera_config
-        self.class.configure_logging
-        yield
+      dir = Dir.mktmpdir('bolt')
+
+      cli = []
+      Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
+        cli << "--#{setting}" << dir
       end
+      Puppet.settings.send(:clear_everything_for_tests)
+      Puppet.initialize_settings(cli)
+      Puppet::GettextConfig.create_default_text_domain
+      Puppet[:trusted_external_command] = @trusted_external
+      Puppet.settings[:hiera_config] = @hiera_config
+      self.class.configure_logging
+      yield
+    ensure
+      # Delete the tmpdir if it still exists. This check is needed to
+      # prevent Bolt from erroring if the tmpdir is somehow deleted
+      # before reaching this point.
+      FileUtils.remove_entry_secure(dir) if File.exist?(dir)
     end
 
     # Parses a snippet of Puppet manifest code and returns the AST represented
@@ -441,13 +450,14 @@ module Bolt
     # Returns a mapping of all modules available to the Bolt compiler
     #
     # @return [Hash{String => Array<Hash{Symbol => String,nil}>}]
-    #   A hash that associates each directory on the module path with an array
+    #   A hash that associates each directory on the modulepath with an array
     #   containing a hash of information for each module in that directory.
     #   The information hash provides the name, version, and a string
     #   indicating whether the module belongs to an internal module group.
     def list_modules
-      internal_module_groups = { BOLTLIB_PATH => 'Plan Language Modules',
-                                 MODULES_PATH => 'Packaged Modules' }
+      internal_module_groups = { Bolt::Config::Modulepath::BOLTLIB_PATH => 'Plan Language Modules',
+                                 Bolt::Config::Modulepath::MODULES_PATH => 'Packaged Modules',
+                                 @project.managed_moduledir.to_s => 'Project Dependencies' }
 
       in_bolt_compiler do
         # NOTE: Can replace map+to_h with transform_values when Ruby 2.4

data/lib/bolt/pal/yaml_plan.rb

@@ -98,10 +98,12 @@ module Bolt
       # subclasses this parent class in order to implement its own evaluation
       # logic.
       class EvaluableString
-        attr_reader :value
+        attr_reader :file, :line, :value
 
-        def initialize(value)
+        def initialize(value, file = nil, line = nil)
           @value = value
+          @file  = file
+          @line  = line
         end
 
         def ==(other)

data/lib/bolt/pal/yaml_plan/evaluator.rb

@@ -191,7 +191,11 @@ module Bolt
               o[key] = evaluate_code_blocks(scope, v)
             end
           when EvaluableString
-            value.evaluate(scope, @evaluator)
+            begin
+              value.evaluate(scope, @evaluator)
+            rescue StandardError => e
+              raise format_evaluate_error(e, value)
+            end
           else
             value
           end
@@ -203,6 +207,24 @@ module Bolt
         def evaluate(value, _scope)
           value
         end
+
+        def format_evaluate_error(error, value)
+          # The Puppet::PreformattedError includes the line number of the
+          # evaluable string that caused the error, while the value includes the
+          # line number of the YAML plan that the string began on. To get the
+          # actual line number of the error, add these two numbers together.
+          line = error.line + value.line
+
+          # If the evaluable string is not a scalar literal, correct for it
+          # being on the same line as the step key.
+          line -= 1 if value.is_a?(BareString)
+
+          Bolt::PlanFailure.new(
+            error.basic_message,
+            'bolt/evaluation-error',
+            { file: value.file, line: line }
+          )
+        end
       end
     end
   end

data/lib/bolt/pal/yaml_plan/loader.rb

@@ -9,10 +9,15 @@ module Bolt
     class YamlPlan
       class Loader
         class PuppetVisitor < Psych::Visitors::NoAliasRuby
-          def self.create_visitor
+          def initialize(scanner, class_loader, file)
+            super(scanner, class_loader)
+            @file = file
+          end
+
+          def self.create_visitor(source_ref)
             class_loader = Psych::ClassLoader::Restricted.new([], [])
             scanner = Psych::ScalarScanner.new(class_loader)
-            new(scanner, class_loader)
+            new(scanner, class_loader, source_ref)
           end
 
           def deserialize(node)
@@ -23,18 +28,18 @@ module Bolt
                 # @ss is a ScalarScanner, from the base ToRuby visitor class
                 node.value
               when Psych::Nodes::Scalar::DOUBLE_QUOTED
-                DoubleQuotedString.new(node.value)
-              # | style string or > style string
-              when Psych::Nodes::Scalar::LITERAL, Psych::Nodes::Scalar::FOLDED
-                CodeLiteral.new(node.value)
-              # This one shouldn't be possible
+                DoubleQuotedString.new(node.value, @file, node.start_line + 1)
+              # | style string
+              when Psych::Nodes::Scalar::LITERAL
+                CodeLiteral.new(node.value, @file, node.start_line + 1)
+              # > style string
               else
                 @ss.tokenize(node.value)
               end
             else
               value = @ss.tokenize(node.value)
               if value.is_a?(String)
-                BareString.new(value)
+                BareString.new(value, @file, node.start_line + 1)
               else
                 value
               end
@@ -50,7 +55,7 @@ module Bolt
                        else
                          Psych.parse(yaml_string, source_ref)
                        end
-          PuppetVisitor.create_visitor.accept(parse_tree)
+          PuppetVisitor.create_visitor(source_ref).accept(parse_tree)
         end
 
         def self.from_string(name, yaml_string, source_ref)

data/lib/bolt/plugin.rb

@@ -168,7 +168,7 @@ module Bolt
     end
 
     def modules
-      @modules ||= Bolt::Module.discover(@pal.modulepath)
+      @modules ||= Bolt::Module.discover(@pal.full_modulepath)
     end
 
     def add_plugin(plugin)

data/lib/bolt/plugin/module.rb

@@ -16,7 +16,7 @@ module Bolt
         mod = modules[name]
         if mod&.plugin?
           opts[:mod] = mod
-          plugin = Bolt::Plugin::Module.new(opts)
+          plugin = Bolt::Plugin::Module.new(**opts)
           plugin.setup
           plugin
         else

data/lib/bolt/project.rb

@@ -16,9 +16,10 @@ module Bolt
                  "These tasks are included in `bolt task show` output"
     }.freeze
 
-    attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
+    attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
-                :deprecations, :downloads, :plans_path
+                :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
+                :backup_dir
 
     def self.default_project(logs = [])
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
@@ -94,15 +95,16 @@ module Bolt
         @deprecations << { type: 'Using bolt.yaml for project configuration', msg: msg }
       end
 
-      @inventory_file = @path + 'inventory.yaml'
-      @modulepath = [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
-      @hiera_config = @path + 'hiera.yaml'
-      @puppetfile = @path + 'Puppetfile'
-      @rerunfile = @path + '.rerun.json'
-      @resource_types = @path + '.resource_types'
-      @type = type
-      @downloads = @path + 'downloads'
-      @plans_path = @path + 'plans'
+      @inventory_file    = @path + 'inventory.yaml'
+      @hiera_config      = @path + 'hiera.yaml'
+      @puppetfile        = @path + 'Puppetfile'
+      @rerunfile         = @path + '.rerun.json'
+      @resource_types    = @path + '.resource_types'
+      @type              = type
+      @downloads         = @path + 'downloads'
+      @plans_path        = @path + 'plans'
+      @managed_moduledir = @path + '.modules'
+      @backup_dir        = @path + '.bolt-bak'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
@@ -112,6 +114,14 @@ module Bolt
 
       @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_OPTIONS.include?(k) }
 
+      # If the 'modules' key is present in the project configuration file,
+      # use the new, shorter modulepath.
+      @modulepath = if @data.key?('modules')
+                      [(@path + 'modules').to_s]
+                    else
+                      [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
+                    end
+
       # Once bolt.yaml deprecation is removed, this attribute should be removed
       # and replaced with .project_file in lib/bolt/config.rb
       @config_file = if (Bolt::Config::BOLT_OPTIONS & @data.keys).any?
@@ -164,7 +174,13 @@ module Bolt
     end
 
     def modules
-      @data['modules']
+      @modules ||= @data['modules']&.map do |mod|
+        if mod.is_a?(String)
+          { 'name' => mod }
+        else
+          mod
+        end
+      end
     end
 
     def validate
@@ -174,7 +190,7 @@ module Bolt
           Invalid project name '#{name}' in bolt-project.yaml; project name must begin with a lowercase letter
           and can include lowercase letters, numbers, and underscores.
           ERROR_STRING
-        elsif Dir.children(Bolt::PAL::BOLTLIB_PATH).include?(name)
+        elsif Dir.children(Bolt::Config::Modulepath::BOLTLIB_PATH).include?(name)
           raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
             "with a built-in Bolt module of the same name."
         end
@@ -194,14 +210,9 @@ module Bolt
           raise Bolt::ValidationError, "'modules' in bolt-project.yaml must be an array"
         end
 
-        @data['modules'].each do |mod|
-          next if mod.is_a?(Hash)
-          raise Bolt::ValidationError, "Module declaration #{mod.inspect} must be a hash"
-        end
-
-        unknown_keys = data['modules'].flat_map(&:keys).uniq - %w[name version_requirement]
-        if unknown_keys.any?
-          @logs << { warn: "Ignoring unknown keys in module declarations: #{unknown_keys.join(', ')}." }
+        @data['modules'].each do |spec|
+          next if spec.is_a?(Hash) || spec.is_a?(String)
+          raise Bolt::ValidationError, "Module specification #{spec.inspect} must be a hash or string"
         end
       end
     end

data/lib/bolt/project_migrator.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'bolt/project_migrator/config'
+require 'bolt/project_migrator/inventory'
+require 'bolt/project_migrator/modules'
+
+module Bolt
+  class ProjectMigrator
+    def initialize(config, outputter)
+      @config    = config
+      @outputter = outputter
+    end
+
+    def migrate
+      unless $stdin.tty?
+        raise Bolt::Error.new(
+          "stdin is not a tty, unable to migrate project",
+          'bolt/stdin-not-a-tty-error'
+        )
+      end
+
+      @outputter.print_message("Migrating project #{@config.project.path}\n\n")
+
+      @outputter.print_action_step(
+        "Migrating a Bolt project may make irreversible changes to the project's "\
+        "configuration and inventory files. Before continuing, make sure the "\
+        "project has a backup or uses a version control system."
+      )
+
+      return 0 unless Bolt::Util.prompt_yes_no("Continue with project migration?", @outputter)
+
+      @outputter.print_message('')
+
+      ok = migrate_inventory && migrate_config && migrate_modules
+
+      if ok
+        @outputter.print_message("Project successfully migrated")
+      else
+        @outputter.print_error("Project could not be migrated completely")
+      end
+
+      ok ? 0 : 1
+    end
+
+    # Migrates the project-level configuration file to the latest version.
+    #
+    private def migrate_config
+      migrator = Bolt::ProjectMigrator::Config.new(@outputter)
+
+      migrator.migrate(
+        @config.project.config_file,
+        @config.project.project_file,
+        @config.inventoryfile || @config.project.inventory_file,
+        @config.project.backup_dir
+      )
+    end
+
+    # Migrates the inventory file to the latest version.
+    #
+    private def migrate_inventory
+      migrator = Bolt::ProjectMigrator::Inventory.new(@outputter)
+
+      migrator.migrate(
+        @config.inventoryfile || @config.project.inventory_file,
+        @config.project.backup_dir
+      )
+    end
+
+    # Migrates the project's modules to use current best practices.
+    #
+    private def migrate_modules
+      migrator = Bolt::ProjectMigrator::Modules.new(@outputter)
+
+      migrator.migrate(
+        @config.project,
+        @config.modulepath
+      )
+    end
+  end
+end