bolt 2.27.0 → 2.32.0

data/lib/bolt/outputter/logger.rb

@@ -13,9 +13,9 @@ module Bolt
       def handle_event(event)
         case event[:type]
         when :step_start
-          log_step_start(event)
+          log_step_start(**event)
         when :step_finish
-          log_step_finish(event)
+          log_step_finish(**event)
         when :plan_start
           log_plan_start(event)
         when :plan_finish

data/lib/bolt/pal.rb

@@ -5,18 +5,16 @@ require 'bolt/executor'
 require 'bolt/error'
 require 'bolt/plan_result'
 require 'bolt/util'
+require 'bolt/config/modulepath'
 require 'etc'
 
 module Bolt
   class PAL
-    BOLTLIB_PATH = File.expand_path('../../bolt-modules', __dir__)
-    MODULES_PATH = File.expand_path('../../modules', __dir__)
-
     # PALError is used to convert errors from executing puppet code into
     # Bolt::Errors
     class PALError < Bolt::Error
       def self.from_preformatted_error(err)
-        if err.cause&.is_a? Bolt::Error
+        if err.cause.is_a? Bolt::Error
           err.cause
         else
           from_error(err)
@@ -29,15 +27,12 @@ module Bolt
         message = err.cause ? err.cause.message : err.message
 
         # Provide the location of an error if it came from a plan
-        details = if defined?(err.file) && err.file
-                    { file:   err.file,
-                      line:   err.line,
-                      column: err.pos }.compact
-                  else
-                    {}
-                  end
+        details = {}
+        details[:file]   = err.file if defined?(err.file)
+        details[:line]   = err.line if defined?(err.line)
+        details[:column] = err.pos if defined?(err.pos)
 
-        e = new(message, details)
+        e = new(message, details.compact)
 
         e.set_backtrace(err.backtrace)
         e
@@ -48,16 +43,16 @@ module Bolt
       end
     end
 
-    attr_reader :modulepath, :user_modulepath
-
     def initialize(modulepath, hiera_config, resource_types, max_compiles = Etc.nprocessors,
                    trusted_external = nil, apply_settings = {}, project = nil)
+      unless modulepath.is_a?(Bolt::Config::Modulepath)
+        msg = "Type error in PAL: modulepath must be a Bolt::Config::Modulepath"
+        raise Bolt::Error.new(msg, "bolt/execution-error")
+      end
       # Nothing works without initialized this global state. Reinitializing
       # is safe and in practice only happens in tests
       self.class.load_puppet
-
-      @user_modulepath = modulepath
-      @modulepath = [BOLTLIB_PATH, *modulepath, MODULES_PATH]
+      @modulepath = modulepath
       @hiera_config = hiera_config
       @trusted_external = trusted_external
       @apply_settings = apply_settings
@@ -66,13 +61,21 @@ module Bolt
       @project = project
 
       @logger = Bolt::Logger.logger(self)
-      if modulepath && !modulepath.empty?
-        @logger.debug("Loading modules from #{@modulepath.join(File::PATH_SEPARATOR)}")
+      unless user_modulepath.empty?
+        @logger.debug("Loading modules from #{full_modulepath.join(File::PATH_SEPARATOR)}")
       end
 
       @loaded = false
     end
 
+    def full_modulepath
+      @modulepath.full_modulepath
+    end
+
+    def user_modulepath
+      @modulepath.user_modulepath
+    end
+
     # Puppet logging is global so this is class method to avoid confusion
     def self.configure_logging
       Puppet::Util::Log.destinations.clear
@@ -160,7 +163,7 @@ module Bolt
     def in_bolt_compiler
       # TODO: If we always call this inside a bolt_executor we can remove this here
       setup
-      r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
+      r = Puppet::Pal.in_tmp_environment('bolt', modulepath: full_modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
         Puppet.override(bolt_project: @project,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
@@ -215,11 +218,11 @@ module Bolt
         apply_executor: applicator || Applicator.new(
           inventory,
           executor,
-          @modulepath,
+          full_modulepath,
           # Skip syncing built-in plugins, since we vendor some Puppet 6
           # versions of "core" types, which are already present on the agent,
           # but may cause issues on Puppet 5 agents.
-          @user_modulepath,
+          user_modulepath,
           @project,
           pdb_client,
           @hiera_config,
@@ -441,13 +444,14 @@ module Bolt
     # Returns a mapping of all modules available to the Bolt compiler
     #
     # @return [Hash{String => Array<Hash{Symbol => String,nil}>}]
-    #   A hash that associates each directory on the module path with an array
+    #   A hash that associates each directory on the modulepath with an array
     #   containing a hash of information for each module in that directory.
     #   The information hash provides the name, version, and a string
     #   indicating whether the module belongs to an internal module group.
     def list_modules
-      internal_module_groups = { BOLTLIB_PATH => 'Plan Language Modules',
-                                 MODULES_PATH => 'Packaged Modules' }
+      internal_module_groups = { Bolt::Config::Modulepath::BOLTLIB_PATH => 'Plan Language Modules',
+                                 Bolt::Config::Modulepath::MODULES_PATH => 'Packaged Modules',
+                                 @project.managed_moduledir.to_s => 'Project Dependencies' }
 
       in_bolt_compiler do
         # NOTE: Can replace map+to_h with transform_values when Ruby 2.4

data/lib/bolt/plugin.rb

@@ -168,7 +168,7 @@ module Bolt
     end
 
     def modules
-      @modules ||= Bolt::Module.discover(@pal.modulepath)
+      @modules ||= Bolt::Module.discover(@pal.full_modulepath)
     end
 
     def add_plugin(plugin)

data/lib/bolt/plugin/module.rb

@@ -16,7 +16,7 @@ module Bolt
         mod = modules[name]
         if mod&.plugin?
           opts[:mod] = mod
-          plugin = Bolt::Plugin::Module.new(opts)
+          plugin = Bolt::Plugin::Module.new(**opts)
           plugin.setup
           plugin
         else

data/lib/bolt/project.rb

@@ -16,9 +16,10 @@ module Bolt
                  "These tasks are included in `bolt task show` output"
     }.freeze
 
-    attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
+    attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
-                :deprecations, :downloads, :plans_path
+                :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
+                :backup_dir
 
     def self.default_project(logs = [])
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
@@ -94,15 +95,16 @@ module Bolt
         @deprecations << { type: 'Using bolt.yaml for project configuration', msg: msg }
       end
 
-      @inventory_file = @path + 'inventory.yaml'
-      @modulepath = [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
-      @hiera_config = @path + 'hiera.yaml'
-      @puppetfile = @path + 'Puppetfile'
-      @rerunfile = @path + '.rerun.json'
-      @resource_types = @path + '.resource_types'
-      @type = type
-      @downloads = @path + 'downloads'
-      @plans_path = @path + 'plans'
+      @inventory_file    = @path + 'inventory.yaml'
+      @hiera_config      = @path + 'hiera.yaml'
+      @puppetfile        = @path + 'Puppetfile'
+      @rerunfile         = @path + '.rerun.json'
+      @resource_types    = @path + '.resource_types'
+      @type              = type
+      @downloads         = @path + 'downloads'
+      @plans_path        = @path + 'plans'
+      @managed_moduledir = @path + '.modules'
+      @backup_dir        = @path + '.bolt-bak'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
@@ -112,6 +114,14 @@ module Bolt
 
       @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_OPTIONS.include?(k) }
 
+      # If the 'modules' key is present in the project configuration file,
+      # use the new, shorter modulepath.
+      @modulepath = if @data.key?('modules')
+                      [(@path + 'modules').to_s]
+                    else
+                      [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
+                    end
+
       # Once bolt.yaml deprecation is removed, this attribute should be removed
       # and replaced with .project_file in lib/bolt/config.rb
       @config_file = if (Bolt::Config::BOLT_OPTIONS & @data.keys).any?
@@ -164,7 +174,13 @@ module Bolt
     end
 
     def modules
-      @data['modules']
+      @modules ||= @data['modules']&.map do |mod|
+        if mod.is_a?(String)
+          { 'name' => mod }
+        else
+          mod
+        end
+      end
     end
 
     def validate
@@ -174,7 +190,7 @@ module Bolt
           Invalid project name '#{name}' in bolt-project.yaml; project name must begin with a lowercase letter
           and can include lowercase letters, numbers, and underscores.
           ERROR_STRING
-        elsif Dir.children(Bolt::PAL::BOLTLIB_PATH).include?(name)
+        elsif Dir.children(Bolt::Config::Modulepath::BOLTLIB_PATH).include?(name)
           raise Bolt::ValidationError, "The project '#{name}' will not be loaded. The project name conflicts "\
             "with a built-in Bolt module of the same name."
         end
@@ -194,15 +210,9 @@ module Bolt
           raise Bolt::ValidationError, "'modules' in bolt-project.yaml must be an array"
         end
 
-        @data['modules'].each do |mod|
-          next if mod.is_a?(Hash)
-          raise Bolt::ValidationError, "Module declaration #{mod.inspect} must be a hash"
-        end
-
-        unknown_keys = data['modules'].flat_map(&:keys).uniq - ['name']
-        if unknown_keys.any?
-          @logs << { warn: "Module declarations in bolt-project.yaml only support a name key. Ignoring "\
-                           "unsupported keys: #{unknown_keys.join(', ')}." }
+        @data['modules'].each do |spec|
+          next if spec.is_a?(Hash) || spec.is_a?(String)
+          raise Bolt::ValidationError, "Module specification #{spec.inspect} must be a hash or string"
         end
       end
     end

data/lib/bolt/project_migrator.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'bolt/project_migrator/config'
+require 'bolt/project_migrator/inventory'
+require 'bolt/project_migrator/modules'
+
+module Bolt
+  class ProjectMigrator
+    def initialize(config, outputter)
+      @config    = config
+      @outputter = outputter
+    end
+
+    def migrate
+      unless $stdin.tty?
+        raise Bolt::Error.new(
+          "stdin is not a tty, unable to migrate project",
+          'bolt/stdin-not-a-tty-error'
+        )
+      end
+
+      @outputter.print_message("Migrating project #{@config.project.path}\n\n")
+
+      @outputter.print_action_step(
+        "Migrating a Bolt project may make irreversible changes to the project's "\
+        "configuration and inventory files. Before continuing, make sure the "\
+        "project has a backup or uses a version control system."
+      )
+
+      return 0 unless Bolt::Util.prompt_yes_no("Continue with project migration?", @outputter)
+
+      @outputter.print_message('')
+
+      ok = migrate_inventory && migrate_config && migrate_modules
+
+      if ok
+        @outputter.print_message("Project successfully migrated")
+      else
+        @outputter.print_error("Project could not be migrated completely")
+      end
+
+      ok ? 0 : 1
+    end
+
+    # Migrates the project-level configuration file to the latest version.
+    #
+    private def migrate_config
+      migrator = Bolt::ProjectMigrator::Config.new(@outputter)
+
+      migrator.migrate(
+        @config.project.config_file,
+        @config.project.project_file,
+        @config.inventoryfile || @config.project.inventory_file,
+        @config.project.backup_dir
+      )
+    end
+
+    # Migrates the inventory file to the latest version.
+    #
+    private def migrate_inventory
+      migrator = Bolt::ProjectMigrator::Inventory.new(@outputter)
+
+      migrator.migrate(
+        @config.inventoryfile || @config.project.inventory_file,
+        @config.project.backup_dir
+      )
+    end
+
+    # Migrates the project's modules to use current best practices.
+    #
+    private def migrate_modules
+      migrator = Bolt::ProjectMigrator::Modules.new(@outputter)
+
+      migrator.migrate(
+        @config.project,
+        @config.modulepath
+      )
+    end
+  end
+end

data/lib/bolt/project_migrator/base.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'bolt/error'
+
+module Bolt
+  class ProjectMigrator
+    class Base
+      def initialize(outputter)
+        @outputter = outputter
+      end
+
+      protected def backup_file(origin_path, backup_dir)
+        unless File.exist?(origin_path)
+          @outputter.print_action_step(
+            "Could not find file #{origin_path}, skipping backup."
+          )
+          return
+        end
+
+        date = Time.new.strftime("%Y%m%d_%H%M%S%L")
+        FileUtils.mkdir_p(backup_dir)
+
+        filename = File.basename(origin_path)
+        backup_path = File.join(backup_dir, "#{filename}.#{date}.bak")
+
+        @outputter.print_action_step(
+          "Backing up #{filename} from #{origin_path} to #{backup_path}"
+        )
+
+        begin
+          FileUtils.cp(origin_path, backup_path)
+        rescue StandardError => e
+          raise Bolt::FileError.new("#{e.message}; unable to create backup of #{filename}.", origin_path)
+        end
+      end
+    end
+  end
+end

data/lib/bolt/project_migrator/config.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'bolt/project_migrator/base'
+
+module Bolt
+  class ProjectMigrator
+    class Config < Base
+      def migrate(config_file, project_file, inventory_file, backup_dir)
+        bolt_yaml_to_bolt_project(config_file, project_file, inventory_file, backup_dir)
+      end
+
+      private def bolt_yaml_to_bolt_project(config_file, project_file, inventory_file, backup_dir)
+        if File.exist?(project_file)
+          return true
+        end
+
+        unless File.exist?(config_file)
+          return true
+        end
+
+        @outputter.print_message "Migrating project configuration\n\n"
+
+        config_data = Bolt::Util.read_optional_yaml_hash(config_file, 'config')
+        transport_data, project_data = config_data.partition do |k, _|
+          Bolt::Config::INVENTORY_OPTIONS.keys.include?(k)
+        end.map(&:to_h)
+
+        if transport_data.any?
+          if File.exist?(inventory_file)
+            inventory_data = Bolt::Util.read_yaml_hash(inventory_file, 'inventory')
+            merged = Bolt::Util.deep_merge(transport_data, inventory_data['config'] || {})
+            inventory_data['config'] = merged
+            backup_file(inventory_file, backup_dir)
+          else
+            FileUtils.touch(inventory_file)
+            inventory_data = { 'config' => transport_data }
+          end
+
+          backup_file(config_file, backup_dir)
+
+          begin
+            @outputter.print_action_step(
+              "Moving transportation configuration options '#{transport_data.keys.join(', ')}' "\
+              "from bolt.yaml to inventory.yaml"
+            )
+
+            File.write(inventory_file, inventory_data.to_yaml)
+            File.write(config_file, project_data.to_yaml)
+          rescue StandardError => e
+            raise Bolt::FileError.new("#{e.message}; unable to write inventory.", inventory_file)
+          end
+        end
+
+        @outputter.print_action_step("Renaming bolt.yaml to bolt-project.yaml")
+        FileUtils.mv(config_file, project_file)
+
+        @outputter.print_action_step(
+          "Successfully migrated config. Please add a 'name' key to bolt-project.yaml "\
+          "to use project-level tasks and plans. Learn more about projects by running "\
+          "'bolt guide project'."
+        )
+
+        true
+      end
+    end
+  end
+end

data/lib/bolt/project_migrator/inventory.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'bolt/project_migrator/base'
+
+module Bolt
+  class ProjectMigrator
+    class Inventory < Base
+      def migrate(inventory_file, backup_dir)
+        inventory_1_to_2(inventory_file, backup_dir)
+      end
+
+      # Migrates an inventory v1 file to inventory v2.
+      #
+      private def inventory_1_to_2(inventory_file, backup_dir)
+        unless File.exist?(inventory_file)
+          return true
+        end
+
+        data = Bolt::Util.read_yaml_hash(inventory_file, 'inventory')
+        data.delete('version') if data['version'] != 2
+        migrated = migrate_group(data)
+
+        return true unless migrated
+
+        @outputter.print_message "Migrating inventory\n\n"
+
+        backup_file(inventory_file, backup_dir)
+
+        begin
+          File.write(inventory_file, data.to_yaml)
+          @outputter.print_action_step(
+            "Successfully migrated Bolt inventory to the latest version."
+          )
+          true
+        rescue StandardError => e
+          raise Bolt::FileError.new(
+            "Unable to write to #{inventory_file}: #{e.message}. See "\
+            "http://pup.pt/bolt-inventory to manually update.",
+            inventory_file
+          )
+        end
+      end
+
+      # Walks an inventory hash and replaces all 'nodes' keys with 'targets'
+      # keys and all 'name' keys nested in a 'targets' hash with 'uri' keys.
+      # Data is modified in place.
+      #
+      private def migrate_group(group)
+        migrated = false
+        if group.key?('nodes')
+          migrated = true
+          targets = group['nodes'].map do |target|
+            target['uri'] = target.delete('name') if target.is_a?(Hash)
+            target
+          end
+          group.delete('nodes')
+          group['targets'] = targets
+        end
+        (group['groups'] || []).each do |subgroup|
+          migrated_group = migrate_group(subgroup)
+          migrated ||= migrated_group
+        end
+        migrated
+      end
+    end
+  end
+end