bolt 2.29.0 → 2.30.0

data/lib/bolt/config.rb

@@ -391,6 +391,12 @@ module Bolt
         @logs << { warn: msg }
       end
 
+      if @project.modules && @data['modulepath']&.include?(@project.managed_moduledir.to_s)
+        raise Bolt::ValidationError,
+              "Found invalid path in modulepath: #{@project.managed_moduledir}. This path "\
+              "is automatically appended to the modulepath and cannot be configured."
+      end
+
       keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
@@ -445,7 +451,13 @@ module Bolt
     end
 
     def modulepath
-      @data['modulepath'] || @project.modulepath
+      path = @data['modulepath'] || @project.modulepath
+
+      if @project.modules
+        path + [@project.managed_moduledir.to_s]
+      else
+        path
+      end
     end
 
     def modulepath=(value)

data/lib/bolt/config/options.rb

@@ -233,7 +233,7 @@ module Bolt
                        "install` command.",
           type: Array,
           items: {
-            type: Hash,
+            type: [Hash, String],
             required: ["name"],
             properties: {
               "name" => {
@@ -250,6 +250,7 @@ module Bolt
           _plugin: false,
           _example: [
             { "name" => "puppetlabs-mysql" },
+            "puppetlabs-facts",
             { "name" => "puppetlabs-apache", "version_requirement" => "5.5.0" },
             { "name" => "puppetlabs-puppetdb", "version_requirement" => "7.x" },
             { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" }

data/lib/bolt/executor.rb

@@ -227,7 +227,7 @@ module Bolt
         data[:resource_mean] = sum / resource_counts.length
       end
 
-      @analytics&.event('Apply', 'ast', data)
+      @analytics&.event('Apply', 'ast', **data)
     end
 
     def report_yaml_plan(plan)

data/lib/bolt/module_installer.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require 'bolt/error'
+require 'bolt/logger'
+
+module Bolt
+  class ModuleInstaller
+    def initialize(outputter, pal)
+      @outputter = outputter
+      @pal       = pal
+      @logger    = Bolt::Logger.logger(self)
+    end
+
+    # Adds a single module to the project.
+    #
+    def add(name, modules, puppetfile_path, moduledir, config_path)
+      require 'bolt/puppetfile'
+
+      # If the project configuration file already includes this module,
+      # exit early.
+      puppetfile  = Bolt::Puppetfile.new(modules)
+      new_module  = Bolt::Puppetfile::Module.from_hash('name' => name)
+
+      if puppetfile.modules.include?(new_module)
+        @outputter.print_message "Project configuration file #{config_path} already "\
+                                 "includes module #{new_module}. Nothing to do."
+        return true
+      end
+
+      # If the Puppetfile exists, make sure it's managed by Bolt.
+      if puppetfile_path.exist?
+        assert_managed_puppetfile(puppetfile, puppetfile_path)
+      end
+
+      # Create a Puppetfile object that includes the new module and its
+      # dependencies. We error early here so we don't add the new module to the
+      # project config or modify the Puppetfile.
+      puppetfile = add_new_module_to_puppetfile(new_module, modules, puppetfile_path)
+
+      # Add the module to the project configuration.
+      @outputter.print_message "Updating project configuration file at #{config_path}"
+
+      data = Bolt::Util.read_yaml_hash(config_path, 'project')
+      data['modules'] ||= []
+      data['modules'] <<  { 'name' => new_module.title }
+
+      begin
+        File.write(config_path, data.to_yaml)
+      rescue SystemCallError => e
+        raise Bolt::FileError.new(
+          "Unable to update project configuration file: #{e.message}",
+          config
+        )
+      end
+
+      # Write the Puppetfile.
+      @outputter.print_message "Writing Puppetfile at #{puppetfile_path}"
+      puppetfile.write(puppetfile_path, moduledir)
+
+      # Install the modules.
+      install_puppetfile(puppetfile_path, moduledir)
+    end
+
+    # Creates a new Puppetfile that includes the new module and its dependencies.
+    #
+    private def add_new_module_to_puppetfile(new_module, modules, path)
+      @outputter.print_message "Resolving module dependencies, this may take a moment"
+
+      # If there is an existing Puppetfile, add the new module and attempt
+      # to resolve. This will not update the versions of any installed modules.
+      if path.exist?
+        puppetfile = Bolt::Puppetfile.parse(path)
+        puppetfile.add_modules(new_module)
+
+        begin
+          puppetfile.resolve
+          return puppetfile
+        rescue Bolt::Error
+          @logger.debug "Unable to find a version of #{new_module} compatible "\
+                        "with installed modules. Attempting to re-resolve modules "\
+                        "from project configuration; some versions of installed "\
+                        "modules may change."
+        end
+      end
+
+      # If there is not an existing Puppetfile, or resolving with pinned
+      # modules fails, resolve all of the module declarations with the new
+      # module.
+      puppetfile = Bolt::Puppetfile.new(modules)
+      puppetfile.add_modules(new_module)
+      puppetfile.resolve
+      puppetfile
+    end
+
+    # Installs a project's module dependencies.
+    #
+    def install(modules, path, moduledir, force: false, resolve: true)
+      require 'bolt/puppetfile'
+
+      puppetfile = Bolt::Puppetfile.new(modules)
+
+      # If the Puppetfile exists, check if it includes specs for each declared
+      # module, erroring if there are any missing. Otherwise, resolve the
+      # module dependencies and write a new Puppetfile. Users can forcibly
+      # overwrite an existing Puppetfile with the '--force' option, or opt to
+      # install the Puppetfile as-is with --no-resolve.
+      #
+      # This is just if resolve is not false (nil should default to true)
+      if resolve != false
+        if path.exist? && !force
+          assert_managed_puppetfile(puppetfile, path)
+        else
+          @outputter.print_message "Resolving module dependencies, this may take a moment"
+          puppetfile.resolve
+
+          @outputter.print_message "Writing Puppetfile at #{path}"
+          # We get here either through 'bolt module install' which uses the
+          # managed modulepath (which isn't configurable) or through bolt
+          # project init --modules, which uses the default modulepath. This
+          # should be safe to assume that if `.modules/` is the moduledir the
+          # user is using the new workflow
+          if moduledir.basename == '.modules'
+            puppetfile.write(path, moduledir)
+          else
+            puppetfile.write(path)
+          end
+        end
+      end
+
+      # Install the modules.
+      install_puppetfile(path, moduledir)
+    end
+
+    # Installs the Puppetfile and generates types.
+    #
+    def install_puppetfile(path, moduledir, config = {})
+      require 'bolt/puppetfile/installer'
+
+      @outputter.print_message "Syncing modules from #{path} to #{moduledir}"
+      ok = Bolt::Puppetfile::Installer.new(config).install(path, moduledir)
+
+      # Automatically generate types after installing modules
+      @pal.generate_types
+
+      @outputter.print_puppetfile_result(ok, path, moduledir)
+
+      ok
+    end
+
+    # Asserts that an existing Puppetfile is managed by Bolt.
+    #
+    private def assert_managed_puppetfile(puppetfile, path)
+      existing_puppetfile = Bolt::Puppetfile.parse(path)
+
+      unless existing_puppetfile.modules.superset? puppetfile.modules
+        missing_modules = puppetfile.modules - existing_puppetfile.modules
+
+        message = <<~MESSAGE.chomp
+          Puppetfile #{path} is missing specifications for the following
+          module declarations:
+
+          #{missing_modules.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
+          
+          This may not be a Puppetfile managed by Bolt. To forcibly overwrite the
+          Puppetfile, run 'bolt module install --force'.
+        MESSAGE
+
+        raise Bolt::Error.new(message, 'bolt/missing-module-specs')
+      end
+    end
+  end
+end

data/lib/bolt/outputter.rb

@@ -35,6 +35,10 @@ module Bolt
       raise NotImplementedError, "print_message() must be implemented by the outputter class"
     end
 
+    def print_error
+      raise NotImplementedError, "print_error() must be implemented by the outputter class"
+    end
+
     def stringify(message)
       formatted = format_message(message)
       if formatted.is_a?(Hash) || formatted.is_a?(Array)

data/lib/bolt/outputter/human.rb

@@ -5,9 +5,12 @@ require 'bolt/pal'
 module Bolt
   class Outputter
     class Human < Bolt::Outputter
-      COLORS = { red: "31",
-                 green: "32",
-                 yellow: "33" }.freeze
+      COLORS = {
+        red:    "31",
+        green:  "32",
+        yellow: "33",
+        cyan:   "36"
+      }.freeze
 
       def print_head; end
 
@@ -31,6 +34,10 @@ module Bolt
         string.sub(/\s\z/, '')
       end
 
+      def wrap(string, width = 80)
+        string.gsub(/(.{1,#{width}})(\s+|\Z)/, "\\1\n")
+      end
+
       def handle_event(event)
         case event[:type]
         when :enable_default_output
@@ -48,9 +55,9 @@ module Bolt
           when :node_result
             print_result(event[:result]) if @verbose
           when :step_start
-            print_step_start(event) if plan_logging?
+            print_step_start(**event) if plan_logging?
           when :step_finish
-            print_step_finish(event) if plan_logging?
+            print_step_finish(**event) if plan_logging?
           when :plan_start
             print_plan_start(event)
           when :plan_finish
@@ -188,7 +195,7 @@ module Bolt
         @stream.puts total_msg
       end
 
-      def print_table(results)
+      def print_table(results, padding_left = 0, padding_right = 3)
         # lazy-load expensive gem code
         require 'terminal-table'
 
@@ -198,8 +205,8 @@ module Bolt
             border_x: '',
             border_y: '',
             border_i: '',
-            padding_left: 0,
-            padding_right: 3,
+            padding_left: padding_left,
+            padding_right: padding_right,
             border_top: false,
             border_bottom: false
           }
@@ -299,9 +306,9 @@ module Bolt
       def print_module_list(module_list)
         module_list.each do |path, modules|
           if (mod = modules.find { |m| m[:internal_module_group] })
-            @stream.puts(mod[:internal_module_group])
+            @stream.puts(colorize(:cyan, mod[:internal_module_group]))
           else
-            @stream.puts(path)
+            @stream.puts(colorize(:cyan, path))
           end
 
           if modules.empty?
@@ -317,7 +324,7 @@ module Bolt
               [m[:name], version]
             end
 
-            print_table(module_info)
+            print_table(module_info, 2, 1)
           end
 
           @stream.write("\n")
@@ -394,6 +401,41 @@ module Bolt
         @stream.puts(message)
       end
 
+      def print_error(message)
+        @stream.puts(colorize(:red, message))
+      end
+
+      def print_prompt(prompt)
+        @stream.print(colorize(:cyan, indent(4, prompt)))
+      end
+
+      def print_prompt_error(message)
+        @stream.puts(colorize(:red, indent(4, message)))
+      end
+
+      def print_migrate_step(step)
+        first, *remaining = wrap(step, 76).lines
+
+        first     = indent(2, "→ #{first}")
+        remaining = remaining.map { |line| indent(4, line) }
+        step      = [first, *remaining, "\n"].join
+
+        @stream.puts(step)
+      end
+
+      def print_migrate_error(error)
+        # Running everything through 'wrap' messes with newlines. Separating
+        # into lines and wrapping each individually ensures separate errors are
+        # distinguishable.
+        first, *remaining = error.lines
+        first = colorize(:red, indent(2, "→ #{wrap(first, 76)}"))
+        wrapped = remaining.map { |l| wrap(l) }
+        to_print = wrapped.map { |line| colorize(:red, indent(4, line)) }
+        step = [first, *to_print, "\n"].join
+
+        @stream.puts(step)
+      end
+
       def duration_to_string(duration)
         hrs = (duration / 3600).floor
         mins = ((duration % 3600) / 60).floor

data/lib/bolt/outputter/json.rb

@@ -97,7 +97,7 @@ module Bolt
       def print_puppetfile_result(success, puppetfile, moduledir)
         @stream.puts({ "success": success,
                        "puppetfile": puppetfile,
-                       "moduledir": moduledir }.to_json)
+                       "moduledir": moduledir.to_s }.to_json)
       end
 
       def print_targets(targets)
@@ -135,6 +135,12 @@ module Bolt
       def print_message(message)
         $stderr.puts(message)
       end
+      alias print_error print_message
+
+      def print_migrate_step(step)
+        $stderr.puts(step)
+      end
+      alias print_migrate_error print_migrate_step
     end
   end
 end

data/lib/bolt/outputter/logger.rb

@@ -13,9 +13,9 @@ module Bolt
       def handle_event(event)
         case event[:type]
         when :step_start
-          log_step_start(event)
+          log_step_start(**event)
         when :step_finish
-          log_step_finish(event)
+          log_step_finish(**event)
         when :plan_start
           log_plan_start(event)
         when :plan_finish

data/lib/bolt/pal.rb

@@ -29,15 +29,12 @@ module Bolt
         message = err.cause ? err.cause.message : err.message
 
         # Provide the location of an error if it came from a plan
-        details = if defined?(err.file) && err.file
-                    { file:   err.file,
-                      line:   err.line,
-                      column: err.pos }.compact
-                  else
-                    {}
-                  end
+        details = {}
+        details[:file]   = err.file if defined?(err.file)
+        details[:line]   = err.line if defined?(err.line)
+        details[:column] = err.pos if defined?(err.pos)
 
-        e = new(message, details)
+        e = new(message, details.compact)
 
         e.set_backtrace(err.backtrace)
         e
@@ -441,13 +438,14 @@ module Bolt
     # Returns a mapping of all modules available to the Bolt compiler
     #
     # @return [Hash{String => Array<Hash{Symbol => String,nil}>}]
-    #   A hash that associates each directory on the module path with an array
+    #   A hash that associates each directory on the modulepath with an array
     #   containing a hash of information for each module in that directory.
     #   The information hash provides the name, version, and a string
     #   indicating whether the module belongs to an internal module group.
     def list_modules
       internal_module_groups = { BOLTLIB_PATH => 'Plan Language Modules',
-                                 MODULES_PATH => 'Packaged Modules' }
+                                 MODULES_PATH => 'Packaged Modules',
+                                 @project.managed_moduledir.to_s => 'Project Dependencies' }
 
       in_bolt_compiler do
         # NOTE: Can replace map+to_h with transform_values when Ruby 2.4

data/lib/bolt/plugin/module.rb

@@ -16,7 +16,7 @@ module Bolt
         mod = modules[name]
         if mod&.plugin?
           opts[:mod] = mod
-          plugin = Bolt::Plugin::Module.new(opts)
+          plugin = Bolt::Plugin::Module.new(**opts)
           plugin.setup
           plugin
         else

data/lib/bolt/project.rb

@@ -16,9 +16,10 @@ module Bolt
                  "These tasks are included in `bolt task show` output"
     }.freeze
 
-    attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
+    attr_reader :path, :data, :config_file, :inventory_file, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :logs, :project_file,
-                :deprecations, :downloads, :plans_path
+                :deprecations, :downloads, :plans_path, :modulepath, :managed_moduledir,
+                :backup_dir
 
     def self.default_project(logs = [])
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user', logs)
@@ -94,15 +95,16 @@ module Bolt
         @deprecations << { type: 'Using bolt.yaml for project configuration', msg: msg }
       end
 
-      @inventory_file = @path + 'inventory.yaml'
-      @modulepath = [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
-      @hiera_config = @path + 'hiera.yaml'
-      @puppetfile = @path + 'Puppetfile'
-      @rerunfile = @path + '.rerun.json'
-      @resource_types = @path + '.resource_types'
-      @type = type
-      @downloads = @path + 'downloads'
-      @plans_path = @path + 'plans'
+      @inventory_file    = @path + 'inventory.yaml'
+      @hiera_config      = @path + 'hiera.yaml'
+      @puppetfile        = @path + 'Puppetfile'
+      @rerunfile         = @path + '.rerun.json'
+      @resource_types    = @path + '.resource_types'
+      @type              = type
+      @downloads         = @path + 'downloads'
+      @plans_path        = @path + 'plans'
+      @managed_moduledir = @path + '.modules'
+      @backup_dir        = @path + '.bolt-bak'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?
@@ -112,6 +114,14 @@ module Bolt
 
       @data = raw_data.reject { |k, _| Bolt::Config::INVENTORY_OPTIONS.include?(k) }
 
+      # If the 'modules' key is present in the project configuration file,
+      # use the new, shorter modulepath.
+      @modulepath = if @data.key?('modules')
+                      [(@path + 'modules').to_s]
+                    else
+                      [(@path + 'modules').to_s, (@path + 'site-modules').to_s, (@path + 'site').to_s]
+                    end
+
       # Once bolt.yaml deprecation is removed, this attribute should be removed
       # and replaced with .project_file in lib/bolt/config.rb
       @config_file = if (Bolt::Config::BOLT_OPTIONS & @data.keys).any?
@@ -164,7 +174,13 @@ module Bolt
     end
 
     def modules
-      @data['modules']
+      @modules ||= @data['modules']&.map do |mod|
+        if mod.is_a?(String)
+          { 'name' => mod }
+        else
+          mod
+        end
+      end
     end
 
     def validate
@@ -195,11 +211,11 @@ module Bolt
         end
 
         @data['modules'].each do |mod|
-          next if mod.is_a?(Hash)
-          raise Bolt::ValidationError, "Module declaration #{mod.inspect} must be a hash"
+          next if (mod.is_a?(Hash) && mod.key?('name')) || mod.is_a?(String)
+          raise Bolt::ValidationError, "Module declaration #{mod.inspect} must be a hash with a name key"
         end
 
-        unknown_keys = data['modules'].flat_map(&:keys).uniq - %w[name version_requirement]
+        unknown_keys = modules.flat_map(&:keys).uniq - %w[name version_requirement]
         if unknown_keys.any?
           @logs << { warn: "Ignoring unknown keys in module declarations: #{unknown_keys.join(', ')}." }
         end