bolt 2.30.0 → 2.34.0

data/lib/bolt/config/modulepath.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'bolt/config'
+
+module Bolt
+  class Config
+    class Modulepath
+      BOLTLIB_PATH = File.expand_path('../../../bolt-modules', __dir__)
+      MODULES_PATH = File.expand_path('../../../modules', __dir__)
+
+      # The user_modulepath only includes the original modulepath and is used during pluginsync.
+      # We don't want to pluginsync any of the content from BOLT_MODULES since that content
+      # includes core modules that can conflict with modules installed with an agent.
+      attr_reader :user_modulepath
+
+      def initialize(user_modulepath, boltlib_path: BOLTLIB_PATH, builtin_content_path: MODULES_PATH)
+        @user_modulepath = Array(user_modulepath).flatten
+        @boltlib_path = Array(boltlib_path).flatten
+        @builtin_content_path = Array(builtin_content_path).flatten
+      end
+
+      # The full_modulepath includes both the BOLTLIB
+      # path and the MODULES_PATH to ensure bolt functions and
+      # built-in content are available in the compliler
+      def full_modulepath
+        @boltlib_path + @user_modulepath + @builtin_content_path
+      end
+    end
+  end
+end

data/lib/bolt/config/options.rb

@@ -234,26 +234,44 @@ module Bolt
           type: Array,
           items: {
             type: [Hash, String],
-            required: ["name"],
-            properties: {
-              "name" => {
-                description: "The name of the module.",
-                type: String
+            oneOf: [
+              {
+                required: ["name"],
+                properties: {
+                  "name" => {
+                    description: "The name of the module.",
+                    type: String
+                  },
+                  "version_requirement" => {
+                    description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
+                                 "shorthand (1.2.x), or a version range (>= 1.2.0).",
+                    type: String
+                  }
+                }
               },
-              "version_requirement" => {
-                description: "The version requirement for the module. Accepts a specific version (1.2.3), version "\
-                             "shorthand (1.2.x), or a version range (>= 1.2.0).",
-                type: String
+              {
+                required: %w[git ref],
+                properties: {
+                  "git" => {
+                    description: "The URL to the public git repository.",
+                    type: String
+                  },
+                  "ref" => {
+                    description: "The git reference to check out. Can be either a branch, tag, or commit SHA.",
+                    type: String
+                  }
+                }
               }
-            }
+            ]
           },
           _plugin: false,
           _example: [
-            { "name" => "puppetlabs-mysql" },
             "puppetlabs-facts",
+            { "name" => "puppetlabs-mysql" },
             { "name" => "puppetlabs-apache", "version_requirement" => "5.5.0" },
             { "name" => "puppetlabs-puppetdb", "version_requirement" => "7.x" },
-            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" }
+            { "name" => "puppetlabs-firewall", "version_requirement" => ">= 1.0.0 < 3.0.0" },
+            { "git" => "https://github.com/puppetlabs/puppetlabs-apt", "ref" => "7.6.0" }
           ]
         },
         "name" => {
@@ -329,7 +347,7 @@ module Bolt
             "server_urls" => {
               description: "An array containing the PuppetDB host to connect to. Include the protocol `https` "\
                            "and the port, which is usually `8081`. For example, "\
-                           "`https://my-master.example.com:8081`.",
+                           "`https://my-puppetdb-server.com:8081`.",
               type: Array,
               _example: ["https://puppet.example.com:8081"]
             },

data/lib/bolt/config/transport/options.rb

@@ -357,7 +357,7 @@ module Bolt
             description: "The URL of the host used for API requests.",
             format: "uri",
             _plugin: true,
-            _example: "https://api.example.com"
+            _example: "https://api.example.com:<port>"
           },
           "shell-command" => {
             type: String,
@@ -374,7 +374,7 @@ module Bolt
           },
           "ssh-command" => {
             type: [Array, String],
-            description: "The command and flags to use when SSHing. This option is used when you need support for "\
+            description: "The command and options to use when SSHing. This option is used when you need support for "\
                          "features or algorithms that are not supported by the net-ssh Ruby library. **This option "\
                          "is experimental.** You can read more about this  option in [Native SSH "\
                          "transport](experimental_features.md#native-ssh-transport).",

data/lib/bolt/error.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'bolt/util'
+
 module Bolt
   class Error < RuntimeError
     attr_reader :kind, :details, :issue_code, :error_code
@@ -24,6 +26,10 @@ module Bolt
       h
     end
 
+    def add_filelineno(details)
+      @details.merge!(details) unless @details['file']
+    end
+
     def to_json(opts = nil)
       to_h.to_json(opts)
     end
@@ -33,13 +39,17 @@ module Bolt
     end
 
     def self.unknown_task(task)
-      new("Could not find a task named \"#{task}\". For a list of available tasks, run \"bolt task show\"",
-          'bolt/unknown-task')
+      command = Bolt::Util.powershell? ? "Get-BoltTask" : "bolt task show"
+      new(
+        "Could not find a task named '#{task}'. For a list of available tasks, run '#{command}'.",
+        'bolt/unknown-task'
+      )
     end
 
     def self.unknown_plan(plan)
+      command = Bolt::Util.powershell? ? "Get-BoltPlan" : "bolt plan show"
       new(
-        "Could not find a plan named \"#{plan}\". For a list of available plans, run \"bolt plan show\"",
+        "Could not find a plan named '#{plan}'. For a list of available plans, run '#{command}'.",
         'bolt/unknown-plan'
       )
     end

data/lib/bolt/executor.rb

@@ -253,33 +253,33 @@ module Bolt
       result
     end
 
-    def run_command(targets, command, options = {})
+    def run_command(targets, command, options = {}, position = [])
       description = options.fetch(:description, "command '#{command}'")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running command '#{command}'", batch) do
-            transport.batch_command(batch, command, options, &method(:publish_event))
+            transport.batch_command(batch, command, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_script(targets, script, arguments, options = {})
+    def run_script(targets, script, arguments, options = {}, position = [])
       description = options.fetch(:description, "script #{script}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running script #{script} with '#{arguments.to_json}'", batch) do
-            transport.batch_script(batch, script, arguments, options, &method(:publish_event))
+            transport.batch_script(batch, script, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task(targets, task, arguments, options = {})
+    def run_task(targets, task, arguments, options = {}, position = [])
       description = options.fetch(:description, "task #{task.name}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
@@ -287,13 +287,25 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch) do
-            transport.batch_task(batch, task, arguments, options, &method(:publish_event))
+            transport.batch_task(batch, task, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task_with(target_mapping, task, options = {})
+    def run_task_with_minimal_logging(targets, task, arguments, options = {})
+      description = options.fetch(:description, "task #{task.name}")
+      log_action(description, targets) do
+        options[:run_as] = run_as if run_as && !options.key?(:run_as)
+        arguments['_task'] = task.name
+
+        batch_execute(targets) do |transport, batch|
+          transport.batch_task(batch, task, arguments, options, [], &method(:publish_event))
+        end
+      end
+    end
+
+    def run_task_with(target_mapping, task, options = {}, position = [])
       targets = target_mapping.keys
       description = options.fetch(:description, "task #{task.name}")
 
@@ -303,26 +315,26 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name}'", batch) do
-            transport.batch_task_with(batch, task, target_mapping, options, &method(:publish_event))
+            transport.batch_task_with(batch, task, target_mapping, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def upload_file(targets, source, destination, options = {})
+    def upload_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file upload from #{source} to #{destination}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Uploading file #{source} to #{destination}", batch) do
-            transport.batch_upload(batch, source, destination, options, &method(:publish_event))
+            transport.batch_upload(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def download_file(targets, source, destination, options = {})
+    def download_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file download from #{source} to #{destination}")
 
       begin
@@ -337,7 +349,7 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Downloading file #{source} to #{destination}", batch) do
-            transport.batch_download(batch, source, destination, options, &method(:publish_event))
+            transport.batch_download(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end

data/lib/bolt/inventory.rb

@@ -56,16 +56,17 @@ module Bolt
         rescue Psych::Exception
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
+      elsif config.inventoryfile
+        data = Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
+        logger.debug("Loaded inventory from #{config.inventoryfile}")
       else
-        data = if config.inventoryfile
-                 Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
-               else
-                 i = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
-                 logger.debug("Loaded inventory from #{config.default_inventoryfile}") if i
-                 i
-               end
-        # This avoids rubocop complaining about identical conditionals
-        logger.debug("Loaded inventory from #{config.inventoryfile}") if config.inventoryfile
+        data = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
+
+        if config.default_inventoryfile.exist?
+          logger.debug("Loaded inventory from #{config.default_inventoryfile}")
+        else
+          logger.debug("Tried to load inventory from #{config.default_inventoryfile}, but the file does not exist")
+        end
       end
 
       # Resolve plugin references from transport config

data/lib/bolt/inventory/group.rb

@@ -241,10 +241,11 @@ module Bolt
         end
 
         if input.key?('nodes')
+          command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
           msg = <<~MSG.chomp
                 Found 'nodes' key in group #{@name}. This looks like a v1 inventory file, which is
                 no longer supported by Bolt. Migrate to a v2 inventory file automatically using
-                'bolt project migrate'.
+                '#{command}'.
                 MSG
           raise ValidationError.new(msg, nil)
         end

data/lib/bolt/module_installer.rb

@@ -2,6 +2,10 @@
 
 require 'bolt/error'
 require 'bolt/logger'
+require 'bolt/module_installer/installer'
+require 'bolt/module_installer/puppetfile'
+require 'bolt/module_installer/resolver'
+require 'bolt/module_installer/specs'
 
 module Bolt
   class ModuleInstaller
@@ -13,36 +17,53 @@ module Bolt
 
     # Adds a single module to the project.
     #
-    def add(name, modules, puppetfile_path, moduledir, config_path)
-      require 'bolt/puppetfile'
-
-      # If the project configuration file already includes this module,
-      # exit early.
-      puppetfile  = Bolt::Puppetfile.new(modules)
-      new_module  = Bolt::Puppetfile::Module.from_hash('name' => name)
-
-      if puppetfile.modules.include?(new_module)
-        @outputter.print_message "Project configuration file #{config_path} already "\
-                                 "includes module #{new_module}. Nothing to do."
+    def add(name, specs, puppetfile_path, moduledir, config_path)
+      project_specs = Specs.new(specs)
+
+      # Exit early if project config already includes a spec with this name.
+      if project_specs.include?(name)
+        @outputter.print_message(
+          "Project configuration file #{config_path} already includes specification with name "\
+          "#{name}. Nothing to do."
+        )
         return true
       end
 
-      # If the Puppetfile exists, make sure it's managed by Bolt.
-      if puppetfile_path.exist?
-        assert_managed_puppetfile(puppetfile, puppetfile_path)
+      @outputter.print_message("Adding module #{name} to project\n\n")
+
+      # Generate the specs to resolve from. If a Puppetfile exists, parse it and
+      # convert the modules to specs. Otherwise, use the project specs.
+      resolve_specs = if puppetfile_path.exist?
+                        existing_puppetfile = Puppetfile.parse(puppetfile_path)
+                        existing_puppetfile.assert_satisfies(project_specs)
+                        Specs.from_puppetfile(existing_puppetfile)
+                      else
+                        project_specs
+                      end
+
+      # Resolve module dependencies. Attempt to first resolve with resolve
+      # specss. If that fails, fall back to resolving from project specs.
+      # This prevents Bolt from modifying installed modules unless there is
+      # a version conflict.
+      @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+
+      begin
+        resolve_specs.add_specs('name' => name)
+        puppetfile = Resolver.new.resolve(resolve_specs)
+      rescue Bolt::Error
+        project_specs.add_specs('name' => name)
+        puppetfile = Resolver.new.resolve(project_specs)
       end
 
-      # Create a Puppetfile object that includes the new module and its
-      # dependencies. We error early here so we don't add the new module to the
-      # project config or modify the Puppetfile.
-      puppetfile = add_new_module_to_puppetfile(new_module, modules, puppetfile_path)
+      # Display the diff between the existing Puppetfile and the new Puppetfile.
+      print_puppetfile_diff(existing_puppetfile, puppetfile)
 
       # Add the module to the project configuration.
-      @outputter.print_message "Updating project configuration file at #{config_path}"
+      @outputter.print_action_step("Updating project configuration file at #{config_path}")
 
       data = Bolt::Util.read_yaml_hash(config_path, 'project')
       data['modules'] ||= []
-      data['modules'] <<  { 'name' => new_module.title }
+      data['modules'] << name.tr('-', '/')
 
       begin
         File.write(config_path, data.to_yaml)
@@ -54,76 +75,104 @@ module Bolt
       end
 
       # Write the Puppetfile.
-      @outputter.print_message "Writing Puppetfile at #{puppetfile_path}"
+      @outputter.print_action_step("Writing Puppetfile at #{puppetfile_path}")
       puppetfile.write(puppetfile_path, moduledir)
 
       # Install the modules.
       install_puppetfile(puppetfile_path, moduledir)
     end
 
-    # Creates a new Puppetfile that includes the new module and its dependencies.
+    # Outputs a diff of an old Puppetfile and a new Puppetfile.
     #
-    private def add_new_module_to_puppetfile(new_module, modules, path)
-      @outputter.print_message "Resolving module dependencies, this may take a moment"
-
-      # If there is an existing Puppetfile, add the new module and attempt
-      # to resolve. This will not update the versions of any installed modules.
-      if path.exist?
-        puppetfile = Bolt::Puppetfile.parse(path)
-        puppetfile.add_modules(new_module)
-
-        begin
-          puppetfile.resolve
-          return puppetfile
-        rescue Bolt::Error
-          @logger.debug "Unable to find a version of #{new_module} compatible "\
-                        "with installed modules. Attempting to re-resolve modules "\
-                        "from project configuration; some versions of installed "\
-                        "modules may change."
+    def print_puppetfile_diff(old, new)
+      # Build hashes mapping the module name to the module object. This makes it
+      # a little easier to determine which modules have been added, removed, or
+      # modified.
+      old = (old&.modules || []).each_with_object({}) do |mod, acc|
+        next unless mod.type == :forge
+        acc[mod.full_name] = mod
+      end
+
+      new = new.modules.each_with_object({}) do |mod, acc|
+        next unless mod.type == :forge
+        acc[mod.full_name] = mod
+      end
+
+      # New modules are those present in new but not in old.
+      added = new.reject { |full_name, _mod| old.include?(full_name) }.values
+
+      if added.any?
+        diff = "Adding the following modules:\n"
+        added.each { |mod| diff += "#{mod.full_name} #{mod.version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Upgraded modules are those that have a newer version in new than old.
+      upgraded = new.select do |full_name, mod|
+        if old.include?(full_name)
+          mod.version > old[full_name].version
+        end
+      end.keys
+
+      if upgraded.any?
+        diff = "Upgrading the following modules:\n"
+        upgraded.each { |full_name| diff += "#{full_name} #{old[full_name].version} to #{new[full_name].version}\n" }
+        @outputter.print_action_step(diff)
+      end
+
+      # Downgraded modules are those that have an older version in new than old.
+      downgraded = new.select do |full_name, mod|
+        if old.include?(full_name)
+          mod.version < old[full_name].version
         end
+      end.keys
+
+      if downgraded.any?
+        diff = "Downgrading the following modules: \n"
+        downgraded.each { |full_name| diff += "#{full_name} #{old[full_name].version} to #{new[full_name].version}\n" }
+        @outputter.print_action_step(diff)
       end
 
-      # If there is not an existing Puppetfile, or resolving with pinned
-      # modules fails, resolve all of the module declarations with the new
-      # module.
-      puppetfile = Bolt::Puppetfile.new(modules)
-      puppetfile.add_modules(new_module)
-      puppetfile.resolve
-      puppetfile
+      # Removed modules are those present in old but not in new.
+      removed = old.reject { |full_name, _mod| new.include?(full_name) }.values
+
+      if removed.any?
+        diff = "Removing the following modules:\n"
+        removed.each { |mod| diff += "#{mod.full_name} #{mod.version}\n" }
+        @outputter.print_action_step(diff)
+      end
     end
 
     # Installs a project's module dependencies.
     #
-    def install(modules, path, moduledir, force: false, resolve: true)
-      require 'bolt/puppetfile'
-
-      puppetfile = Bolt::Puppetfile.new(modules)
-
-      # If the Puppetfile exists, check if it includes specs for each declared
-      # module, erroring if there are any missing. Otherwise, resolve the
-      # module dependencies and write a new Puppetfile. Users can forcibly
-      # overwrite an existing Puppetfile with the '--force' option, or opt to
-      # install the Puppetfile as-is with --no-resolve.
-      #
-      # This is just if resolve is not false (nil should default to true)
+    def install(specs, path, moduledir, force: false, resolve: true)
+      @outputter.print_message("Installing project modules\n\n")
+
       if resolve != false
-        if path.exist? && !force
-          assert_managed_puppetfile(puppetfile, path)
-        else
-          @outputter.print_message "Resolving module dependencies, this may take a moment"
-          puppetfile.resolve
+        specs = Specs.new(specs)
+
+        # If forcibly installing or if there is no Puppetfile, resolve
+        # and write a Puppetfile.
+        if force || !path.exist?
+          @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+          puppetfile = Resolver.new.resolve(specs)
 
-          @outputter.print_message "Writing Puppetfile at #{path}"
           # We get here either through 'bolt module install' which uses the
           # managed modulepath (which isn't configurable) or through bolt
           # project init --modules, which uses the default modulepath. This
           # should be safe to assume that if `.modules/` is the moduledir the
           # user is using the new workflow
-          if moduledir.basename == '.modules'
+          @outputter.print_action_step("Writing Puppetfile at #{path}")
+          if moduledir.basename.to_s == '.modules'
             puppetfile.write(path, moduledir)
           else
             puppetfile.write(path)
           end
+        # If not forcibly installing and there is a Puppetfile, assert
+        # that it satisfies the specs.
+        else
+          puppetfile = Puppetfile.parse(path)
+          puppetfile.assert_satisfies(specs)
         end
       end
 
@@ -134,39 +183,16 @@ module Bolt
     # Installs the Puppetfile and generates types.
     #
     def install_puppetfile(path, moduledir, config = {})
-      require 'bolt/puppetfile/installer'
-
-      @outputter.print_message "Syncing modules from #{path} to #{moduledir}"
-      ok = Bolt::Puppetfile::Installer.new(config).install(path, moduledir)
+      @outputter.print_action_step("Syncing modules from #{path} to #{moduledir}")
+      ok = Installer.new(config).install(path, moduledir)
 
       # Automatically generate types after installing modules
+      @outputter.print_action_step("Generating type references")
       @pal.generate_types
 
       @outputter.print_puppetfile_result(ok, path, moduledir)
 
       ok
     end
-
-    # Asserts that an existing Puppetfile is managed by Bolt.
-    #
-    private def assert_managed_puppetfile(puppetfile, path)
-      existing_puppetfile = Bolt::Puppetfile.parse(path)
-
-      unless existing_puppetfile.modules.superset? puppetfile.modules
-        missing_modules = puppetfile.modules - existing_puppetfile.modules
-
-        message = <<~MESSAGE.chomp
-          Puppetfile #{path} is missing specifications for the following
-          module declarations:
-
-          #{missing_modules.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
-          
-          This may not be a Puppetfile managed by Bolt. To forcibly overwrite the
-          Puppetfile, run 'bolt module install --force'.
-        MESSAGE
-
-        raise Bolt::Error.new(message, 'bolt/missing-module-specs')
-      end
-    end
   end
 end