bolt 2.32.0 → 2.36.0

data/lib/bolt/config/transport/base.rb

@@ -2,6 +2,7 @@
 
 require 'bolt/error'
 require 'bolt/util'
+require 'bolt/config/validator'
 require 'bolt/config/transport/options'
 
 module Bolt
@@ -90,6 +91,14 @@ module Bolt
           self::OPTIONS
         end
 
+        def self.schema
+          {
+            type:       Hash,
+            properties: self::TRANSPORT_OPTIONS.slice(*self::OPTIONS),
+            _plugin:    true
+          }
+        end
+
         private def defaults
           unless defined? self.class::DEFAULTS
             raise NotImplementedError,
@@ -116,25 +125,7 @@ module Bolt
 
         # Validation defaults to just asserting the option types
         private def validate
-          assert_type
-        end
-
-        # Validates that each option is the correct type. Types are loaded from the TRANSPORT_OPTIONS hash.
-        private def assert_type
-          @config.each_pair do |opt, val|
-            types = Array(TRANSPORT_OPTIONS.dig(opt, :type)).compact
-
-            next if val.nil? || types.empty? || types.include?(val.class)
-
-            # Ruby doesn't have a Boolean class, so add it to the types list if TrueClass or FalseClass
-            # are present.
-            if types.include?(TrueClass) || types.include?(FalseClass)
-              types = types - [TrueClass, FalseClass] + ['Boolean']
-            end
-
-            raise Bolt::ValidationError,
-                  "#{opt} must be of type #{types.join(', ')}; received #{val.class} #{val.inspect} "
-          end
+          Bolt::Config::Validator.new.validate(@config.compact, self.class.schema.fetch(:properties))
         end
       end
     end

data/lib/bolt/config/transport/local.rb

@@ -29,13 +29,6 @@ module Bolt
           if @config['interpreters']
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
-
-          if (run_as_cmd = @config['run-as-command'])
-            unless run_as_cmd.all? { |n| n.is_a?(String) }
-              raise Bolt::ValidationError,
-                    "run-as-command must be an Array of Strings, received #{run_as_cmd.class} #{run_as_cmd.inspect}"
-            end
-          end
         end
       end
     end

data/lib/bolt/config/transport/options.rb

@@ -357,7 +357,7 @@ module Bolt
             description: "The URL of the host used for API requests.",
             format: "uri",
             _plugin: true,
-            _example: "https://api.example.com"
+            _example: "https://api.example.com:8143"
           },
           "shell-command" => {
             type: String,

data/lib/bolt/config/transport/ssh.rb

@@ -73,6 +73,14 @@ module Bolt
           %w[ssh-command native-ssh].concat(OPTIONS)
         end
 
+        def self.schema
+          {
+            type:       Hash,
+            properties: self::TRANSPORT_OPTIONS.slice(*(self::OPTIONS + self::NATIVE_OPTIONS)),
+            _plugin:    true
+          }
+        end
+
         private def filter(unfiltered)
           # Because we filter before merging config together it's impossible to
           # know whether both ssh-command *and* native-ssh will be specified
@@ -87,12 +95,6 @@ module Bolt
           super
 
           if (key_opt = @config['private-key'])
-            unless key_opt.instance_of?(String) || (key_opt.instance_of?(Hash) && key_opt.include?('key-data'))
-              raise Bolt::ValidationError,
-                    "private-key option must be a path to a private key file or a Hash containing the 'key-data', "\
-                    "received #{key_opt.class} #{key_opt}"
-            end
-
             if key_opt.instance_of?(String)
               @config['private-key'] = File.expand_path(key_opt, @project)
 
@@ -114,14 +116,6 @@ module Bolt
                   "Unsupported login-shell #{@config['login-shell']}. Supported shells are #{LOGIN_SHELLS.join(', ')}"
           end
 
-          %w[encryption-algorithms host-key-algorithms kex-algorithms mac-algorithms run-as-command].each do |opt|
-            next unless @config.key?(opt)
-            unless @config[opt].all? { |n| n.is_a?(String) }
-              raise Bolt::ValidationError,
-                    "#{opt} must be an Array of Strings, received #{@config[opt].inspect}"
-            end
-          end
-
           if @config['login-shell'] == 'powershell'
             %w[tty run-as].each do |key|
               if @config[key]

data/lib/bolt/config/validator.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require 'bolt/error'
+
+# This class validates config against a schema, raising an error that includes
+# details about any invalid configuration.
+#
+module Bolt
+  class Config
+    class Validator
+      attr_reader :deprecations, :warnings
+
+      def initialize
+        @errors       = []
+        @deprecations = []
+        @warnings     = []
+        @path         = []
+      end
+
+      # This is the entry method for validating data against the schema.
+      # It loops over each key-value pair in the data hash and validates
+      # the value against the relevant schema definition.
+      #
+      def validate(data, schema, location = nil)
+        @location = location
+
+        validate_keys(data.keys, schema.keys)
+
+        data.each_pair do |key, value|
+          next unless schema.key?(key)
+
+          @path.push(key)
+
+          check_deprecated(key, schema[key], location)
+          validate_value(value, schema[key])
+        ensure
+          @path.pop
+        end
+
+        raise_error
+      end
+
+      # Adds a warning if the given option is deprecated.
+      #
+      def check_deprecated(key, definition, location)
+        if definition.key?(:_deprecation)
+          message  = "Option '#{path}' "
+          message += "at #{location} " if location
+          message += "is deprecated. #{definition[:_deprecation]}"
+          @deprecations << { option: key, message: message }
+        end
+      end
+
+      # Raises a ValidationError if there are any errors. All error messages
+      # created during validation are concatenated into a single error
+      # message.
+      #
+      private def raise_error
+        return unless @errors.any?
+
+        message = "Invalid configuration"
+        message += " at #{@location}" if @location
+        message += ":\n"
+        message += @errors.map { |error| "\s\s#{error}" }.join("\n")
+
+        raise Bolt::ValidationError, message
+      end
+
+      # Validate an individual value. This performs validation that is
+      # common to all values, including type validation. After validating
+      # the value's type, the value is passed off to an individual
+      # validation method for the value's type.
+      #
+      private def validate_value(value, definition)
+        return if plugin_reference?(value, definition)
+        return unless valid_type?(value, definition)
+
+        case value
+        when Hash
+          validate_hash(value, definition)
+        when Array
+          validate_array(value, definition)
+        when String
+          validate_string(value, definition)
+        when Numeric
+          validate_number(value, definition)
+        end
+      end
+
+      # Validates a hash value, logging errors for any validations that fail.
+      # This will enumerate each key-value pair in the hash and validate each
+      # value individually.
+      #
+      private def validate_hash(value, definition)
+        properties = definition[:properties] ? definition[:properties].keys : []
+
+        if definition[:properties] && definition[:additionalProperties].nil?
+          validate_keys(value.keys, properties)
+        end
+
+        if definition[:required] && (definition[:required] - value.keys).any?
+          missing = definition[:required] - value.keys
+          @errors << "Value at '#{path}' is missing required keys #{missing.join(', ')}"
+        end
+
+        value.each_pair do |key, val|
+          @path.push(key)
+
+          if properties.include?(key)
+            validate_value(val, definition[:properties][key])
+          elsif definition[:additionalProperties]
+            validate_value(val, definition[:additionalProperties])
+          end
+        ensure
+          @path.pop
+        end
+      end
+
+      # Validates an array value, logging errors for any validations that fail.
+      # This will enumerate the items in the array and validate each item
+      # individually.
+      #
+      private def validate_array(value, definition)
+        if definition[:uniqueItems] && value.size != value.uniq.size
+          @errors << "Value at '#{path}' must not include duplicate elements"
+          return
+        end
+
+        return unless definition.key?(:items)
+
+        value.each_with_index do |item, index|
+          @path.push(index)
+          validate_value(item, definition[:items])
+        ensure
+          @path.pop
+        end
+      end
+
+      # Validates a string value, logging errors for any validations that fail.
+      #
+      private def validate_string(value, definition)
+        if definition.key?(:enum) && !definition[:enum].include?(value)
+          message  = "Value at '#{path}' must be "
+          message += "one of " if definition[:enum].count > 1
+          message += definition[:enum].join(', ')
+          multitype_error(message, value, definition)
+        end
+      end
+
+      # Validates a numeric value, logging errors for any validations that fail.
+      #
+      private def validate_number(value, definition)
+        if definition.key?(:minimum) && value < definition[:minimum]
+          @errors << "Value at '#{path}' must be a minimum of #{definition[:minimum]}"
+        end
+      end
+
+      # Adds warnings for unknown config options.
+      #
+      private def validate_keys(keys, known_keys)
+        (keys - known_keys).each do |key|
+          message  = "Unknown option '#{key}'"
+          message += " at '#{path}'" if @path.any?
+          message += " at #{@location}" if @location
+          message += "."
+          @warnings << message
+        end
+      end
+
+      # Returns true if a value is a plugin reference. This also validates whether
+      # a value can be a plugin reference in the first place. If the value is a
+      # plugin reference but cannot be one according to the schema, then this will
+      # log an error.
+      #
+      private def plugin_reference?(value, definition)
+        if value.is_a?(Hash) && value.key?('_plugin')
+          unless definition[:_plugin]
+            @errors << "Value at '#{path}' is a plugin reference, which is unsupported at "\
+                       "this location"
+          end
+
+          true
+        else
+          false
+        end
+      end
+
+      # Asserts the type for each option against the type specified in the schema
+      # definition. The schema definition can specify multiple valid types, so the
+      # value needs to only match one of the types to be valid. Returns early if
+      # there is no type in the definition (in practice this shouldn't happen, but
+      # this will safeguard against any dev mistakes).
+      #
+      private def valid_type?(value, definition)
+        return unless definition.key?(:type)
+
+        types = Array(definition[:type])
+
+        if types.include?(value.class)
+          true
+        else
+          if types.include?(TrueClass) || types.include?(FalseClass)
+            types = types - [TrueClass, FalseClass] + ['Boolean']
+          end
+
+          @errors << "Value at '#{path}' must be of type #{types.join(' or ')}"
+
+          false
+        end
+      end
+
+      # Adds an error that includes additional helpful information for values
+      # that accept multiple types.
+      #
+      private def multitype_error(message, value, definition)
+        if Array(definition[:type]).count > 1
+          types    = Array(definition[:type]) - [value.class]
+          message += " or must be of type #{types.join(' or ')}"
+        end
+
+        @errors << message
+      end
+
+      # Returns the formatted path for the key.
+      #
+      private def path
+        @path.join('.')
+      end
+    end
+  end
+end

data/lib/bolt/error.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'bolt/util'
+
 module Bolt
   class Error < RuntimeError
     attr_reader :kind, :details, :issue_code, :error_code
@@ -24,6 +26,10 @@ module Bolt
       h
     end
 
+    def add_filelineno(details)
+      @details.merge!(details) unless @details['file']
+    end
+
     def to_json(opts = nil)
       to_h.to_json(opts)
     end
@@ -33,13 +39,17 @@ module Bolt
     end
 
     def self.unknown_task(task)
-      new("Could not find a task named \"#{task}\". For a list of available tasks, run \"bolt task show\"",
-          'bolt/unknown-task')
+      command = Bolt::Util.powershell? ? "Get-BoltTask" : "bolt task show"
+      new(
+        "Could not find a task named '#{task}'. For a list of available tasks, run '#{command}'.",
+        'bolt/unknown-task'
+      )
     end
 
     def self.unknown_plan(plan)
+      command = Bolt::Util.powershell? ? "Get-BoltPlan" : "bolt plan show"
       new(
-        "Could not find a plan named \"#{plan}\". For a list of available plans, run \"bolt plan show\"",
+        "Could not find a plan named '#{plan}'. For a list of available plans, run '#{command}'.",
         'bolt/unknown-plan'
       )
     end
@@ -80,6 +90,20 @@ module Bolt
     end
   end
 
+  class ParallelFailure < Bolt::Error
+    def initialize(results, failed_indices)
+      details = {
+        'action' => 'parallelize',
+        'failed_indices' => failed_indices,
+        'results' => results
+      }
+      message = "Plan aborted: parallel block failed on #{failed_indices.length} target"
+      message += "s" unless failed_indices.length == 1
+      super(message, 'bolt/parallel-failure', details)
+      @error_code = 2
+    end
+  end
+
   class PlanFailure < Error
     def initialize(*args)
       super(*args)
@@ -121,6 +145,16 @@ module Bolt
     end
   end
 
+  class InvalidParallelResult < Error
+    def initialize(result_str, file, line)
+      super("Parallel block returned an invalid result: #{result_str}",
+            'bolt/invalid-plan-result',
+            { 'file' => file,
+              'line' => line,
+              'result_string' => result_str })
+    end
+  end
+
   class ValidationError < Bolt::Error
     def initialize(msg)
       super(msg, 'bolt/validation-error')

data/lib/bolt/executor.rb

@@ -17,6 +17,7 @@ require 'bolt/transport/orch'
 require 'bolt/transport/local'
 require 'bolt/transport/docker'
 require 'bolt/transport/remote'
+require 'bolt/yarn'
 
 module Bolt
   TRANSPORTS = {
@@ -29,7 +30,7 @@ module Bolt
   }.freeze
 
   class Executor
-    attr_reader :noop, :transports
+    attr_reader :noop, :transports, :in_parallel
     attr_accessor :run_as
 
     def initialize(concurrency = 1,
@@ -60,6 +61,7 @@ module Bolt
 
       @noop = noop
       @run_as = nil
+      @in_parallel = false
       @pool = if concurrency > 0
                 Concurrent::ThreadPoolExecutor.new(name: 'exec', max_threads: concurrency)
               else
@@ -84,6 +86,14 @@ module Bolt
       self
     end
 
+    def unsubscribe(subscriber, types = nil)
+      if types.nil? || types.sort == @subscribers[subscriber]&.sort
+        @subscribers.delete(subscriber)
+      elsif @subscribers[subscriber].is_a?(Array)
+        @subscribers[subscriber] = @subscribers[subscriber] - types
+      end
+    end
+
     def publish_event(event)
       @subscribers.each do |subscriber, types|
         # If types isn't set or if the subscriber is subscribed to
@@ -246,54 +256,54 @@ module Bolt
       @logger.trace { "Failed to submit analytics event: #{e.message}" }
     end
 
-    def with_node_logging(description, batch)
-      @logger.info("#{description} on #{batch.map(&:safe_name)}")
+    def with_node_logging(description, batch, log_level = :info)
+      @logger.send(log_level, "#{description} on #{batch.map(&:safe_name)}")
       result = yield
-      @logger.info(result.to_json)
+      @logger.send(log_level, result.to_json)
       result
     end
 
-    def run_command(targets, command, options = {})
+    def run_command(targets, command, options = {}, position = [])
       description = options.fetch(:description, "command '#{command}'")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running command '#{command}'", batch) do
-            transport.batch_command(batch, command, options, &method(:publish_event))
+            transport.batch_command(batch, command, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_script(targets, script, arguments, options = {})
+    def run_script(targets, script, arguments, options = {}, position = [])
       description = options.fetch(:description, "script #{script}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running script #{script} with '#{arguments.to_json}'", batch) do
-            transport.batch_script(batch, script, arguments, options, &method(:publish_event))
+            transport.batch_script(batch, script, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task(targets, task, arguments, options = {})
+    def run_task(targets, task, arguments, options = {}, position = [], log_level = :info)
       description = options.fetch(:description, "task #{task.name}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
         arguments['_task'] = task.name
 
         batch_execute(targets) do |transport, batch|
-          with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch) do
-            transport.batch_task(batch, task, arguments, options, &method(:publish_event))
+          with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch, log_level) do
+            transport.batch_task(batch, task, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task_with(target_mapping, task, options = {})
+    def run_task_with(target_mapping, task, options = {}, position = [])
       targets = target_mapping.keys
       description = options.fetch(:description, "task #{task.name}")
 
@@ -303,26 +313,26 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name}'", batch) do
-            transport.batch_task_with(batch, task, target_mapping, options, &method(:publish_event))
+            transport.batch_task_with(batch, task, target_mapping, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def upload_file(targets, source, destination, options = {})
+    def upload_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file upload from #{source} to #{destination}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Uploading file #{source} to #{destination}", batch) do
-            transport.batch_upload(batch, source, destination, options, &method(:publish_event))
+            transport.batch_upload(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def download_file(targets, source, destination, options = {})
+    def download_file(targets, source, destination, options = {}, position = [])
       description = options.fetch(:description, "file download from #{source} to #{destination}")
 
       begin
@@ -337,7 +347,7 @@ module Bolt
 
         batch_execute(targets) do |transport, batch|
           with_node_logging("Downloading file #{source} to #{destination}", batch) do
-            transport.batch_download(batch, source, destination, options, &method(:publish_event))
+            transport.batch_download(batch, source, destination, options, position, &method(:publish_event))
           end
         end
       end
@@ -347,6 +357,82 @@ module Bolt
       plan.call_by_name_with_scope(scope, params, true)
     end
 
+    def create_yarn(scope, block, object, index)
+      fiber = Fiber.new do
+        # Create the new scope
+        newscope = Puppet::Parser::Scope.new(scope.compiler)
+        local = Puppet::Parser::Scope::LocalScope.new
+
+        # Compress the current scopes into a single vars hash to add to the new scope
+        current_scope = scope.effective_symtable(true)
+        until current_scope.nil?
+          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
+          current_scope = current_scope.parent
+        end
+        newscope.push_ephemerals([local])
+
+        begin
+          result = catch(:return) do
+            args = { block.parameters[0][1].to_s => object }
+            block.closure.call_by_name_with_scope(newscope, args, true)
+          end
+
+          # If we got a return from the block, get it's value
+          # Otherwise the result is the last line from the block
+          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+
+          # Validate the result is a PlanResult
+          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
+            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
+          end
+
+          result
+        rescue Puppet::PreformattedError => e
+          if e.cause.is_a?(Bolt::Error)
+            e.cause
+          else
+            raise e
+          end
+        end
+      end
+
+      Bolt::Yarn.new(fiber, index)
+    end
+
+    def handle_event(event)
+      case event[:type]
+      when :node_result
+        @thread_completed = true
+      end
+    end
+
+    def round_robin(skein)
+      subscribe(self, [:node_result])
+      results = Array.new(skein.length)
+      @in_parallel = true
+
+      until skein.empty?
+        @thread_completed = false
+        r = nil
+
+        skein.each do |yarn|
+          if yarn.alive?
+            r = yarn.resume
+          else
+            results[yarn.index] = yarn.value
+            skein.delete(yarn)
+          end
+        end
+
+        next unless r == 'unfinished'
+        sleep(0.1) until @thread_completed || skein.empty?
+      end
+
+      @in_parallel = false
+      unsubscribe(self, [:node_result])
+      results
+    end
+
     class TimeoutError < RuntimeError; end
 
     def wait_until_available(targets,