bolt 2.29.0 → 2.33.2

data/lib/bolt/config.rb

@@ -391,6 +391,12 @@ module Bolt
         @logs << { warn: msg }
       end
 
+      if @project.modules && @data['modulepath']&.include?(@project.managed_moduledir.to_s)
+        raise Bolt::ValidationError,
+              "Found invalid path in modulepath: #{@project.managed_moduledir}. This path "\
+              "is automatically appended to the modulepath and cannot be configured."
+      end
+
       keys = OPTIONS.keys - %w[plugins plugin_hooks puppetdb]
       keys.each do |key|
         next unless Bolt::Util.references?(@data[key])
@@ -445,7 +451,13 @@ module Bolt
     end
 
     def modulepath
-      @data['modulepath'] || @project.modulepath
+      path = @data['modulepath'] || @project.modulepath
+
+      if @project.modules
+        path + [@project.managed_moduledir.to_s]
+      else
+        path
+      end
     end
 
     def modulepath=(value)

data/lib/bolt/config/modulepath.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'bolt/config'
+
+module Bolt
+  class Config
+    class Modulepath
+      BOLTLIB_PATH = File.expand_path('../../../bolt-modules', __dir__)
+      MODULES_PATH = File.expand_path('../../../modules', __dir__)
+
+      # The user_modulepath only includes the original modulepath and is used during pluginsync.
+      # We don't want to pluginsync any of the content from BOLT_MODULES since that content
+      # includes core modules that can conflict with modules installed with an agent.
+      attr_reader :user_modulepath
+
+      def initialize(user_modulepath, boltlib_path: BOLTLIB_PATH, builtin_content_path: MODULES_PATH)
+        @user_modulepath = Array(user_modulepath).flatten
+        @boltlib_path = Array(boltlib_path).flatten
+        @builtin_content_path = Array(builtin_content_path).flatten
+      end
+
+      # The full_modulepath includes both the BOLTLIB
+      # path and the MODULES_PATH to ensure bolt functions and
+      # built-in content are available in the compliler
+      def full_modulepath
+        @boltlib_path + @user_modulepath + @builtin_content_path
+      end
+    end
+  end
+end

data/lib/bolt/config/options.rb

@@ -233,26 +233,45 @@ module Bolt
                        "install` command.",
           type: Array,
           items: {
-            type: Hash,
-            required: ["name"],
-            properties: {
-              "name" => {
-                description: "The name of the module.",
-                type: String
+            type: [Hash, String],
+            oneOf: [
+              {
+                required: ["name"],
+                properties: {
+                  "name" => {
+                    description: "The name of the module.",
+                    type: String
+                  },
+                  "version_requirement" => {
+                    description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
+                                 "shorthand (1.2.x), or a version range (>= 1.2.0).",
+                    type: String
+                  }
+                }
               },
-              "version_requirement" => {
-                description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
-                             "shorthand (1.2.x), or a version range (>= 1.2.0).",
-                type: String
+              {
+                required: %w[git ref],
+                properties: {
+                  "git" => {
+                    description: "The URL to the public git repository.",
+                    type: String
+                  },
+                  "ref" => {
+                    description: "The git reference to check out. Can be either a branch, tag, or commit SHA.",
+                    type: String
+                  }
+                }
               }
-            }
+            ]
           },
           _plugin: false,
           _example: [
+            "puppetlabs-facts",
             { "name" => "puppetlabs-mysql" },
             { "name" => "puppetlabs-apache", "version_requirement" => "5.5.0" },
             { "name" => "puppetlabs-puppetdb", "version_requirement" => "7.x" },
-            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" }
+            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" },
+            { "git" => "https://github.com/puppetlabs/puppetlabs-apt", "ref" => "7.6.0" }
           ]
         },
         "name" => {
@@ -328,7 +347,7 @@ module Bolt
             "server_urls" => {
               description: "An array containing the PuppetDB host to connect to. Include the protocol `https` "\
                            "and the port, which is usually `8081`. For example, "\
-                           "`https://my-master.example.com:8081`.",
+                           "`https://my-puppetdb-server.com:8081`.",
               type: Array,
               _example: ["https://puppet.example.com:8081"]
             },

data/lib/bolt/config/transport/options.rb

@@ -357,7 +357,7 @@ module Bolt
             description: "The URL of the host used for API requests.",
             format: "uri",
             _plugin: true,
-            _example: "https://api.example.com"
+            _example: "https://api.example.com:<port>"
           },
           "shell-command" => {
             type: String,
@@ -374,7 +374,7 @@ module Bolt
           },
           "ssh-command" => {
             type: [Array, String],
-            description: "The command and flags to use when SSHing. This option is used when you need support for "\
+            description: "The command and options to use when SSHing. This option is used when you need support for "\
                          "features or algorithms that are not supported by the net-ssh Ruby library. **This option "\
                          "is experimental.** You can read more about this  option in [Native SSH "\
                          "transport](experimental_features.md#native-ssh-transport).",

data/lib/bolt/error.rb

@@ -24,6 +24,10 @@ module Bolt
       h
     end
 
+    def add_filelineno(details)
+      @details.merge!(details) unless @details['file']
+    end
+
     def to_json(opts = nil)
       to_h.to_json(opts)
     end

data/lib/bolt/executor.rb

@@ -227,7 +227,7 @@ module Bolt
         data[:resource_mean] = sum / resource_counts.length
       end
 
-      @analytics&.event('Apply', 'ast', data)
+      @analytics&.event('Apply', 'ast', **data)
     end
 
     def report_yaml_plan(plan)
@@ -253,33 +253,33 @@ module Bolt
       result
     end
 
-    def run_command(targets, command, options = {})
+    def run_command(targets, command, options = {}, position = [])
       description = options.fetch(:description, "command '#{command}'")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running command '#{command}'", batch) do
-            transport.batch_command(batch, command, options, &method(:publish_event))
+            transport.batch_command(batch, command, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_script(targets, script, arguments, options = {})
+    def run_script(targets, script, arguments, options = {}, position = [])
       description = options.fetch(:description, "script #{script}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running script #{script} with '#{arguments.to_json}'", batch) do
-            transport.batch_script(batch, script, arguments, options, &method(:publish_event))
+            transport.batch_script(batch, script, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task(targets, task, arguments, options = {})
+    def run_task(targets, task, arguments, options = {}, position = [])
       description = options.fetch(:description, "task #{task.name}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
@@ -287,13 +287,13 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch) do
-            transport.batch_task(batch, task, arguments, options, &method(:publish_event))
+            transport.batch_task(batch, task, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task_with(target_mapping, task, options = {})
+    def run_task_with(target_mapping, task, options = {}, position = [])
       targets = target_mapping.keys
       description = options.fetch(:description, "task #{task.name}")
 
@@ -303,26 +303,26 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name}'", batch) do
-            transport.batch_task_with(batch, task, target_mapping, options, &method(:publish_event))
+            transport.batch_task_with(batch, task, target_mapping, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def upload_file(targets, source, destination, options = {})
+    def upload_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file upload from #{source} to #{destination}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Uploading file #{source} to #{destination}", batch) do
-            transport.batch_upload(batch, source, destination, options, &method(:publish_event))
+            transport.batch_upload(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def download_file(targets, source, destination, options = {})
+    def download_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file download from #{source} to #{destination}")
 
       begin
@@ -337,7 +337,7 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Downloading file #{source} to #{destination}", batch) do
-            transport.batch_download(batch, source, destination, options, &method(:publish_event))
+            transport.batch_download(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end

data/lib/bolt/inventory.rb

@@ -56,16 +56,17 @@ module Bolt
         rescue Psych::Exception
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
+      elsif config.inventoryfile
+        data = Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
+        logger.debug("Loaded inventory from #{config.inventoryfile}")
       else
-        data = if config.inventoryfile
-                 Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
-               else
-                 i = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
-                 logger.debug("Loaded inventory from #{config.default_inventoryfile}") if i
-                 i
-               end
-        # This avoids rubocop complaining about identical conditionals
-        logger.debug("Loaded inventory from #{config.inventoryfile}") if config.inventoryfile
+        data = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+
+        if config.default_inventoryfile.exist?
+          logger.debug("Loaded inventory from #{config.default_inventoryfile}")
+        else
+          logger.debug("Tried to load inventory from #{config.default_inventoryfile}, but the file does not exist")
+        end
       end
 
       # Resolve plugin references from transport config

data/lib/bolt/module_installer.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'bolt/error'
+require 'bolt/logger'
+require 'bolt/module_installer/installer'
+require 'bolt/module_installer/puppetfile'
+require 'bolt/module_installer/resolver'
+require 'bolt/module_installer/specs'
+
+module Bolt
+  class ModuleInstaller
+    def initialize(outputter, pal)
+      @outputter = outputter
+      @pal       = pal
+      @logger    = Bolt::Logger.logger(self)
+    end
+
+    # Adds a single module to the project.
+    #
+    def add(name, specs, puppetfile_path, moduledir, config_path)
+      project_specs = Specs.new(specs)
+
+      # Exit early if project config already includes a spec with this name.
+      if project_specs.include?(name)
+        @outputter.print_message(
+          "Project configuration file #{config_path} already includes specification with name "\
+          "#{name}. Nothing to do."
+        )
+        return true
+      end
+
+      @outputter.print_message("Adding module #{name} to project\n\n")
+
+      # Generate the specs to resolve from. If a Puppetfile exists, parse it and
+      # convert the modules to specs. Otherwise, use the project specs.
+      resolve_specs = if puppetfile_path.exist?
+                        existing_puppetfile = Puppetfile.parse(puppetfile_path)
+                        existing_puppetfile.assert_satisfies(project_specs)
+                        Specs.from_puppetfile(existing_puppetfile)
+                      else
+                        project_specs
+                      end
+
+      # Resolve module dependencies. Attempt to first resolve with resolve
+      # specss. If that fails, fall back to resolving from project specs.
+      # This prevents Bolt from modifying installed modules unless there is
+      # a version conflict.
+      @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+
+      begin
+        resolve_specs.add_specs('name' => name)
+        puppetfile = Resolver.new.resolve(resolve_specs)
+      rescue Bolt::Error
+        project_specs.add_specs('name' => name)
+        puppetfile = Resolver.new.resolve(project_specs)
+      end
+
+      # Display the diff between the existing Puppetfile and the new Puppetfile.
+      print_puppetfile_diff(existing_puppetfile, puppetfile)
+
+      # Add the module to the project configuration.
+      @outputter.print_action_step("Updating project configuration file at #{config_path}")
+
+      data = Bolt::Util.read_yaml_hash(config_path, 'project')
+      data['modules'] ||= []
+      data['modules'] << name.tr('-', '/')
+
+      begin
+        File.write(config_path, data.to_yaml)
+      rescue SystemCallError => e
+        raise Bolt::FileError.new(
+          "Unable to update project configuration file: #{e.message}",
+          config
+        )
+      end
+
+      # Write the Puppetfile.
+      @outputter.print_action_step("Writing Puppetfile at #{puppetfile_path}")
+      puppetfile.write(puppetfile_path, moduledir)
+
+      # Install the modules.
+      install_puppetfile(puppetfile_path, moduledir)
+    end
+
+    # Outputs a diff of an old Puppetfile and a new Puppetfile.
+    #
+    def print_puppetfile_diff(old, new)
+      # Build hashes mapping the module name to the module object. This makes it
+      # a little easier to determine which modules have been added, removed, or
+      # modified.
+      old = (old&.modules || []).each_with_object({}) do |mod, acc|
+        next unless mod.type == :forge
+        acc[mod.full_name] = mod
+      end
+
+      new = new.modules.each_with_object({}) do |mod, acc|
+        next unless mod.type == :forge
+        acc[mod.full_name] = mod
+      end
+
+      # New modules are those present in new but not in old.
+      added = new.reject { |full_name, _mod| old.include?(full_name) }.values
+
+      if added.any?
+        diff = "Adding the following modules:\n"
+        added.each { |mod| diff += "#{mod.full_name} #{mod.version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Upgraded modules are those that have a newer version in new than old.
+      upgraded = new.select do |full_name, mod|
+        if old.include?(full_name)
+          mod.version > old[full_name].version
+        end
+      end.keys
+
+      if upgraded.any?
+        diff = "Upgrading the following modules:\n"
+        upgraded.each { |full_name| diff += "#{full_name} #{old[full_name].version} to #{new[full_name].version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Downgraded modules are those that have an older version in new than old.
+      downgraded = new.select do |full_name, mod|
+        if old.include?(full_name)
+          mod.version < old[full_name].version
+        end
+      end.keys
+
+      if downgraded.any?
+        diff = "Downgrading the following modules: \n"
+        downgraded.each { |full_name| diff += "#{full_name} #{old[full_name].version} to #{new[full_name].version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Removed modules are those present in old but not in new.
+      removed = old.reject { |full_name, _mod| new.include?(full_name) }.values
+
+      if removed.any?
+        diff = "Removing the following modules:\n"
+        removed.each { |mod| diff += "#{mod.full_name} #{mod.version}\n" }
+        @outputter.print_action_step(diff)
+      end
+    end
+
+    # Installs a project's module dependencies.
+    #
+    def install(specs, path, moduledir, force: false, resolve: true)
+      @outputter.print_message("Installing project modules\n\n")
+
+      if resolve != false
+        specs = Specs.new(specs)
+
+        # If forcibly installing or if there is no Puppetfile, resolve
+        # and write a Puppetfile.
+        if force || !path.exist?
+          @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+          puppetfile = Resolver.new.resolve(specs)
+
+          # We get here either through 'bolt module install' which uses the
+          # managed modulepath (which isn't configurable) or through bolt
+          # project init --modules, which uses the default modulepath. This
+          # should be safe to assume that if `.modules/` is the moduledir the
+          # user is using the new workflow
+          @outputter.print_action_step("Writing Puppetfile at #{path}")
+          if moduledir.basename.to_s == '.modules'
+            puppetfile.write(path, moduledir)
+          else
+            puppetfile.write(path)
+          end
+        # If not forcibly installing and there is a Puppetfile, assert
+        # that it satisfies the specs.
+        else
+          puppetfile = Puppetfile.parse(path)
+          puppetfile.assert_satisfies(specs)
+        end
+      end
+
+      # Install the modules.
+      install_puppetfile(path, moduledir)
+    end
+
+    # Installs the Puppetfile and generates types.
+    #
+    def install_puppetfile(path, moduledir, config = {})
+      @outputter.print_action_step("Syncing modules from #{path} to #{moduledir}")
+      ok = Installer.new(config).install(path, moduledir)
+
+      # Automatically generate types after installing modules
+      @outputter.print_action_step("Generating type references")
+      @pal.generate_types
+
+      @outputter.print_puppetfile_result(ok, path, moduledir)
+
+      ok
+    end
+  end
+end