bolt 2.31.0 → 2.35.0

data/lib/bolt/config/transport/options.rb

@@ -357,7 +357,7 @@ module Bolt
             description: "The URL of the host used for API requests.",
             format: "uri",
             _plugin: true,
-            _example: "https://api.example.com"
+            _example: "https://api.example.com:8143"
           },
           "shell-command" => {
             type: String,

data/lib/bolt/error.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'bolt/util'
+
 module Bolt
   class Error < RuntimeError
     attr_reader :kind, :details, :issue_code, :error_code
@@ -24,6 +26,10 @@ module Bolt
       h
     end
 
+    def add_filelineno(details)
+      @details.merge!(details) unless @details['file']
+    end
+
     def to_json(opts = nil)
       to_h.to_json(opts)
     end
@@ -33,13 +39,17 @@ module Bolt
     end
 
     def self.unknown_task(task)
-      new("Could not find a task named \"#{task}\". For a list of available tasks, run \"bolt task show\"",
-          'bolt/unknown-task')
+      command = Bolt::Util.powershell? ? "Get-BoltTask" : "bolt task show"
+      new(
+        "Could not find a task named '#{task}'. For a list of available tasks, run '#{command}'.",
+        'bolt/unknown-task'
+      )
     end
 
     def self.unknown_plan(plan)
+      command = Bolt::Util.powershell? ? "Get-BoltPlan" : "bolt plan show"
       new(
-        "Could not find a plan named \"#{plan}\". For a list of available plans, run \"bolt plan show\"",
+        "Could not find a plan named '#{plan}'. For a list of available plans, run '#{command}'.",
         'bolt/unknown-plan'
       )
     end
@@ -80,6 +90,20 @@ module Bolt
     end
   end
 
+  class ParallelFailure < Bolt::Error
+    def initialize(results, failed_indices)
+      details = {
+        'action' => 'parallelize',
+        'failed_indices' => failed_indices,
+        'results' => results
+      }
+      message = "Plan aborted: parallel block failed on #{failed_indices.length} target"
+      message += "s" unless failed_indices.length == 1
+      super(message, 'bolt/parallel-failure', details)
+      @error_code = 2
+    end
+  end
+
   class PlanFailure < Error
     def initialize(*args)
       super(*args)
@@ -121,6 +145,16 @@ module Bolt
     end
   end
 
+  class InvalidParallelResult < Error
+    def initialize(result_str, file, line)
+      super("Parallel block returned an invalid result: #{result_str}",
+            'bolt/invalid-plan-result',
+            { 'file' => file,
+              'line' => line,
+              'result_string' => result_str })
+    end
+  end
+
   class ValidationError < Bolt::Error
     def initialize(msg)
       super(msg, 'bolt/validation-error')

data/lib/bolt/executor.rb

@@ -17,6 +17,7 @@ require 'bolt/transport/orch'
 require 'bolt/transport/local'
 require 'bolt/transport/docker'
 require 'bolt/transport/remote'
+require 'bolt/yarn'
 
 module Bolt
   TRANSPORTS = {
@@ -29,7 +30,7 @@ module Bolt
   }.freeze
 
   class Executor
-    attr_reader :noop, :transports
+    attr_reader :noop, :transports, :in_parallel
     attr_accessor :run_as
 
     def initialize(concurrency = 1,
@@ -60,6 +61,7 @@ module Bolt
 
       @noop = noop
       @run_as = nil
+      @in_parallel = false
       @pool = if concurrency > 0
                 Concurrent::ThreadPoolExecutor.new(name: 'exec', max_threads: concurrency)
               else
@@ -84,6 +86,14 @@ module Bolt
       self
     end
 
+    def unsubscribe(subscriber, types = nil)
+      if types.nil? || types.sort == @subscribers[subscriber]&.sort
+        @subscribers.delete(subscriber)
+      elsif @subscribers[subscriber].is_a?(Array)
+        @subscribers[subscriber] = @subscribers[subscriber] - types
+      end
+    end
+
     def publish_event(event)
       @subscribers.each do |subscriber, types|
         # If types isn't set or if the subscriber is subscribed to
@@ -253,33 +263,33 @@ module Bolt
       result
     end
 
-    def run_command(targets, command, options = {})
+    def run_command(targets, command, options = {}, position = [])
       description = options.fetch(:description, "command '#{command}'")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running command '#{command}'", batch) do
-            transport.batch_command(batch, command, options, &method(:publish_event))
+            transport.batch_command(batch, command, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_script(targets, script, arguments, options = {})
+    def run_script(targets, script, arguments, options = {}, position = [])
       description = options.fetch(:description, "script #{script}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running script #{script} with '#{arguments.to_json}'", batch) do
-            transport.batch_script(batch, script, arguments, options, &method(:publish_event))
+            transport.batch_script(batch, script, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task(targets, task, arguments, options = {})
+    def run_task(targets, task, arguments, options = {}, position = [])
       description = options.fetch(:description, "task #{task.name}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
@@ -287,13 +297,25 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch) do
-            transport.batch_task(batch, task, arguments, options, &method(:publish_event))
+            transport.batch_task(batch, task, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task_with(target_mapping, task, options = {})
+    def run_task_with_minimal_logging(targets, task, arguments, options = {})
+      description = options.fetch(:description, "task #{task.name}")
+      log_action(description, targets) do
+        options[:run_as] = run_as if run_as && !options.key?(:run_as)
+        arguments['_task'] = task.name
+
+        batch_execute(targets) do |transport, batch|
+          transport.batch_task(batch, task, arguments, options, [], &method(:publish_event))
+        end
+      end
+    end
+
+    def run_task_with(target_mapping, task, options = {}, position = [])
       targets = target_mapping.keys
       description = options.fetch(:description, "task #{task.name}")
 
@@ -303,26 +325,26 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name}'", batch) do
-            transport.batch_task_with(batch, task, target_mapping, options, &method(:publish_event))
+            transport.batch_task_with(batch, task, target_mapping, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def upload_file(targets, source, destination, options = {})
+    def upload_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file upload from #{source} to #{destination}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Uploading file #{source} to #{destination}", batch) do
-            transport.batch_upload(batch, source, destination, options, &method(:publish_event))
+            transport.batch_upload(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def download_file(targets, source, destination, options = {})
+    def download_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file download from #{source} to #{destination}")
 
       begin
@@ -337,7 +359,7 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Downloading file #{source} to #{destination}", batch) do
-            transport.batch_download(batch, source, destination, options, &method(:publish_event))
+            transport.batch_download(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end
@@ -347,6 +369,82 @@ module Bolt
       plan.call_by_name_with_scope(scope, params, true)
     end
 
+    def create_yarn(scope, block, object, index)
+      fiber = Fiber.new do
+        # Create the new scope
+        newscope = Puppet::Parser::Scope.new(scope.compiler)
+        local = Puppet::Parser::Scope::LocalScope.new
+
+        # Compress the current scopes into a single vars hash to add to the new scope
+        current_scope = scope.effective_symtable(true)
+        until current_scope.nil?
+          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
+          current_scope = current_scope.parent
+        end
+        newscope.push_ephemerals([local])
+
+        begin
+          result = catch(:return) do
+            args = { block.parameters[0][1].to_s => object }
+            block.closure.call_by_name_with_scope(newscope, args, true)
+          end
+
+          # If we got a return from the block, get it's value
+          # Otherwise the result is the last line from the block
+          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+
+          # Validate the result is a PlanResult
+          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
+            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
+          end
+
+          result
+        rescue Puppet::PreformattedError => e
+          if e.cause.is_a?(Bolt::Error)
+            e.cause
+          else
+            raise e
+          end
+        end
+      end
+
+      Bolt::Yarn.new(fiber, index)
+    end
+
+    def handle_event(event)
+      case event[:type]
+      when :node_result
+        @thread_completed = true
+      end
+    end
+
+    def round_robin(skein)
+      subscribe(self, [:node_result])
+      results = Array.new(skein.length)
+      @in_parallel = true
+
+      until skein.empty?
+        @thread_completed = false
+        r = nil
+
+        skein.each do |yarn|
+          if yarn.alive?
+            r = yarn.resume
+          else
+            results[yarn.index] = yarn.value
+            skein.delete(yarn)
+          end
+        end
+
+        next unless r == 'unfinished'
+        sleep(0.1) until @thread_completed || skein.empty?
+      end
+
+      @in_parallel = false
+      unsubscribe(self, [:node_result])
+      results
+    end
+
     class TimeoutError < RuntimeError; end
 
     def wait_until_available(targets,

data/lib/bolt/inventory/group.rb

@@ -241,10 +241,11 @@ module Bolt
         end
 
         if input.key?('nodes')
+          command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
           msg = <<~MSG.chomp
                 Found 'nodes' key in group #{@name}. This looks like a v1 inventory file, which is
                 no longer supported by Bolt. Migrate to a v2 inventory file automatically using
-                'bolt project migrate'.
+                '#{command}'.
                 MSG
           raise ValidationError.new(msg, nil)
         end

data/lib/bolt/module_installer.rb

@@ -2,6 +2,10 @@
 
 require 'bolt/error'
 require 'bolt/logger'
+require 'bolt/module_installer/installer'
+require 'bolt/module_installer/puppetfile'
+require 'bolt/module_installer/resolver'
+require 'bolt/module_installer/specs'
 
 module Bolt
   class ModuleInstaller
@@ -13,45 +17,53 @@ module Bolt
 
     # Adds a single module to the project.
     #
-    def add(name, modules, puppetfile_path, moduledir, config_path)
-      require 'bolt/puppetfile'
-
-      @outputter.print_message("Adding module #{name} to project\n\n")
-
-      # If the project configuration file already includes this module,
-      # exit early.
-      puppetfile  = Bolt::Puppetfile.new(modules)
-      new_module  = Bolt::Puppetfile::Module.from_hash('name' => name)
-
-      if puppetfile.modules.include?(new_module)
-        @outputter.print_action_step(
-          "Project configuration file #{config_path} already includes module #{new_module}. Nothing to do."
+    def add(name, specs, puppetfile_path, moduledir, config_path)
+      project_specs = Specs.new(specs)
+
+      # Exit early if project config already includes a spec with this name.
+      if project_specs.include?(name)
+        @outputter.print_message(
+          "Project configuration file #{config_path} already includes specification with name "\
+          "#{name}. Nothing to do."
         )
         return true
       end
 
-      # If the Puppetfile exists, make sure it's managed by Bolt.
-      if puppetfile_path.exist?
-        assert_managed_puppetfile(puppetfile, puppetfile_path)
-        existing = Bolt::Puppetfile.parse(puppetfile_path)
-      else
-        existing = Bolt::Puppetfile.new
-      end
+      @outputter.print_message("Adding module #{name} to project\n\n")
 
-      # Create a Puppetfile object that includes the new module and its
-      # dependencies. We error early here so we don't add the new module to the
-      # project config or modify the Puppetfile.
-      puppetfile = add_new_module_to_puppetfile(new_module, modules, puppetfile_path)
+      # Generate the specs to resolve from. If a Puppetfile exists, parse it and
+      # convert the modules to specs. Otherwise, use the project specs.
+      resolve_specs = if puppetfile_path.exist?
+                        existing_puppetfile = Puppetfile.parse(puppetfile_path)
+                        existing_puppetfile.assert_satisfies(project_specs)
+                        Specs.from_puppetfile(existing_puppetfile)
+                      else
+                        project_specs
+                      end
+
+      # Resolve module dependencies. Attempt to first resolve with resolve
+      # specss. If that fails, fall back to resolving from project specs.
+      # This prevents Bolt from modifying installed modules unless there is
+      # a version conflict.
+      @outputter.print_action_step("Resolving module dependencies, this may take a moment")
+
+      begin
+        resolve_specs.add_specs('name' => name)
+        puppetfile = Resolver.new.resolve(resolve_specs)
+      rescue Bolt::Error
+        project_specs.add_specs('name' => name)
+        puppetfile = Resolver.new.resolve(project_specs)
+      end
 
       # Display the diff between the existing Puppetfile and the new Puppetfile.
-      print_puppetfile_diff(existing, puppetfile)
+      print_puppetfile_diff(existing_puppetfile, puppetfile)
 
       # Add the module to the project configuration.
       @outputter.print_action_step("Updating project configuration file at #{config_path}")
 
       data = Bolt::Util.read_yaml_hash(config_path, 'project')
       data['modules'] ||= []
-      data['modules'] <<  { 'name' => new_module.title }
+      data['modules'] << name.tr('-', '/')
 
       begin
         File.write(config_path, data.to_yaml)
@@ -70,130 +82,97 @@ module Bolt
       install_puppetfile(puppetfile_path, moduledir)
     end
 
-    # Creates a new Puppetfile that includes the new module and its dependencies.
-    #
-    private def add_new_module_to_puppetfile(new_module, modules, path)
-      @outputter.print_action_step("Resolving module dependencies, this may take a moment")
-
-      # If there is an existing Puppetfile, add the new module and attempt
-      # to resolve. This will not update the versions of any installed modules.
-      if path.exist?
-        puppetfile = Bolt::Puppetfile.parse(path)
-        puppetfile.add_modules(new_module)
-
-        begin
-          puppetfile.resolve
-          return puppetfile
-        rescue Bolt::Error
-          @logger.debug "Unable to find a version of #{new_module} compatible "\
-                        "with installed modules. Attempting to re-resolve modules "\
-                        "from project configuration; some versions of installed "\
-                        "modules may change."
-        end
-      end
-
-      # If there is not an existing Puppetfile, or resolving with pinned
-      # modules fails, resolve all of the module declarations with the new
-      # module.
-      puppetfile = Bolt::Puppetfile.new(modules)
-      puppetfile.add_modules(new_module)
-      puppetfile.resolve
-      puppetfile
-    end
-
     # Outputs a diff of an old Puppetfile and a new Puppetfile.
     #
     def print_puppetfile_diff(old, new)
-      # Build hashes mapping the module title to the module object. This makes it
+      # Build hashes mapping the module name to the module object. This makes it
       # a little easier to determine which modules have been added, removed, or
       # modified.
-      old = old.modules.each_with_object({}) do |mod, acc|
-        acc[mod.title] = mod
+      old = (old&.modules || []).each_with_object({}) do |mod, acc|
+        next unless mod.type == :forge
+        acc[mod.full_name] = mod
       end
 
       new = new.modules.each_with_object({}) do |mod, acc|
-        acc[mod.title] = mod
+        next unless mod.type == :forge
+        acc[mod.full_name] = mod
       end
 
       # New modules are those present in new but not in old.
-      added = new.reject { |title, _mod| old.include?(title) }.values
+      added = new.reject { |full_name, _mod| old.include?(full_name) }.values
 
       if added.any?
         diff = "Adding the following modules:\n"
-        added.each { |mod| diff += "#{mod.title} #{mod.version}\n" }
+        added.each { |mod| diff += "#{mod.full_name} #{mod.version}\n" }
         @outputter.print_action_step(diff)
       end
 
       # Upgraded modules are those that have a newer version in new than old.
-      upgraded = new.select do |title, mod|
-        if old.include?(title)
-          SemanticPuppet::Version.parse(mod.version) > SemanticPuppet::Version.parse(old[title].version)
+      upgraded = new.select do |full_name, mod|
+        if old.include?(full_name)
+          mod.version > old[full_name].version
         end
       end.keys
 
       if upgraded.any?
         diff = "Upgrading the following modules:\n"
-        upgraded.each { |title| diff += "#{title} #{old[title].version} to #{new[title].version}\n" }
+        upgraded.each { |full_name| diff += "#{full_name} #{old[full_name].version} to #{new[full_name].version}\n" }
         @outputter.print_action_step(diff)
       end
 
       # Downgraded modules are those that have an older version in new than old.
-      downgraded = new.select do |title, mod|
-        if old.include?(title)
-          SemanticPuppet::Version.parse(mod.version) < SemanticPuppet::Version.parse(old[title].version)
+      downgraded = new.select do |full_name, mod|
+        if old.include?(full_name)
+          mod.version < old[full_name].version
         end
       end.keys
 
       if downgraded.any?
         diff = "Downgrading the following modules: \n"
-        downgraded.each { |title| diff += "#{title} #{old[title].version} to #{new[title].version}\n" }
+        downgraded.each { |full_name| diff += "#{full_name} #{old[full_name].version} to #{new[full_name].version}\n" }
         @outputter.print_action_step(diff)
       end
 
       # Removed modules are those present in old but not in new.
-      removed = old.reject { |title, _mod| new.include?(title) }.values
+      removed = old.reject { |full_name, _mod| new.include?(full_name) }.values
 
       if removed.any?
         diff = "Removing the following modules:\n"
-        removed.each { |mod| diff += "#{mod.title} #{mod.version}\n" }
+        removed.each { |mod| diff += "#{mod.full_name} #{mod.version}\n" }
         @outputter.print_action_step(diff)
       end
     end
 
     # Installs a project's module dependencies.
     #
-    def install(modules, path, moduledir, force: false, resolve: true)
-      require 'bolt/puppetfile'
-
+    def install(specs, path, moduledir, force: false, resolve: true)
       @outputter.print_message("Installing project modules\n\n")
 
-      puppetfile = Bolt::Puppetfile.new(modules)
-
-      # If the Puppetfile exists, check if it includes specs for each declared
-      # module, erroring if there are any missing. Otherwise, resolve the
-      # module dependencies and write a new Puppetfile. Users can forcibly
-      # overwrite an existing Puppetfile with the '--force' option, or opt to
-      # install the Puppetfile as-is with --no-resolve.
-      #
-      # This is just if resolve is not false (nil should default to true)
       if resolve != false
-        if path.exist? && !force
-          assert_managed_puppetfile(puppetfile, path)
-        else
+        specs = Specs.new(specs)
+
+        # If forcibly installing or if there is no Puppetfile, resolve
+        # and write a Puppetfile.
+        if force || !path.exist?
           @outputter.print_action_step("Resolving module dependencies, this may take a moment")
-          puppetfile.resolve
+          puppetfile = Resolver.new.resolve(specs)
 
-          @outputter.print_action_step("Writing Puppetfile at #{path}")
           # We get here either through 'bolt module install' which uses the
           # managed modulepath (which isn't configurable) or through bolt
           # project init --modules, which uses the default modulepath. This
           # should be safe to assume that if `.modules/` is the moduledir the
           # user is using the new workflow
+          @outputter.print_action_step("Writing Puppetfile at #{path}")
           if moduledir.basename.to_s == '.modules'
             puppetfile.write(path, moduledir)
           else
             puppetfile.write(path)
           end
+        # If not forcibly installing and there is a Puppetfile, assert
+        # that it satisfies the specs.
+        else
+          puppetfile = Puppetfile.parse(path)
+          puppetfile.assert_satisfies(specs)
         end
       end
 
@@ -204,39 +183,16 @@ module Bolt
     # Installs the Puppetfile and generates types.
     #
     def install_puppetfile(path, moduledir, config = {})
-      require 'bolt/puppetfile/installer'
-
       @outputter.print_action_step("Syncing modules from #{path} to #{moduledir}")
-      ok = Bolt::Puppetfile::Installer.new(config).install(path, moduledir)
+      ok = Installer.new(config).install(path, moduledir)
 
       # Automatically generate types after installing modules
+      @outputter.print_action_step("Generating type references")
       @pal.generate_types
 
       @outputter.print_puppetfile_result(ok, path, moduledir)
 
       ok
     end
-
-    # Asserts that an existing Puppetfile is managed by Bolt.
-    #
-    private def assert_managed_puppetfile(puppetfile, path)
-      existing_puppetfile = Bolt::Puppetfile.parse(path)
-
-      unless existing_puppetfile.modules.superset? puppetfile.modules
-        missing_modules = puppetfile.modules - existing_puppetfile.modules
-
-        message = <<~MESSAGE.chomp
-          Puppetfile #{path} is missing specifications for the following
-          module declarations:
-
-          #{missing_modules.map(&:to_hash).to_yaml.lines.drop(1).join.chomp}
-          
-          This may not be a Puppetfile managed by Bolt. To forcibly overwrite the
-          Puppetfile, run 'bolt module install --force'.
-        MESSAGE
-
-        raise Bolt::Error.new(message, 'bolt/missing-module-specs')
-      end
-    end
   end
 end