bolt 2.34.0 → 2.40.1

data/lib/bolt/config/transport/local.rb

@@ -8,6 +8,7 @@ module Bolt
     module Transport
       class Local < Base
         WINDOWS_OPTIONS = %w[
+          bundled-ruby
           cleanup
           interpreters
           tmpdir
@@ -29,13 +30,6 @@ module Bolt
           if @config['interpreters']
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
-
-          if (run_as_cmd = @config['run-as-command'])
-            unless run_as_cmd.all? { |n| n.is_a?(String) }
-              raise Bolt::ValidationError,
-                    "run-as-command must be an Array of Strings, received #{run_as_cmd.class} #{run_as_cmd.inspect}"
-            end
-          end
         end
       end
     end

data/lib/bolt/config/transport/options.rb

@@ -6,73 +6,8 @@ module Bolt
       module Options
         LOGIN_SHELLS = %w[sh bash zsh dash ksh powershell].freeze
 
-        # The following constants define the various configuration options available to Bolt's
-        # transports. Each constant is a hash where keys are the configuration option and values
-        # are the option's definition. These options are used in multiple locations:
-        #
-        #   - Automatic type validation when loading and setting configuration
-        #   - Generating reference documentation for configuration files
-        #   - Generating JSON schemas for configuration files
-        #
-        # Data includes keys defined by JSON Schema Draft 07 as well as some metadata used
-        # by Bolt to generate documentation. The following keys are used:
-        #
-        #   :description    String      A detailed description of the option and what it does. This
-        #                               field is used in both documentation and the JSON schemas,
-        #                               and should provide as much detail as possible, including
-        #                               links to relevant documentation.
-        #
-        #   :type           Class       The expected type of a value. These should be Ruby classes,
-        #                               as this field is used to perform automatic type validation.
-        #                               If an option can accept more than one type, this should be
-        #                               an array of types. Boolean values should set :type to
-        #                               [TrueClass, FalseClass], as Ruby does not have a single
-        #                               Boolean class.
-        #
-        #   :items          Hash        A definition hash for items in an array. Similar to values
-        #                               for top-level options, items can have a :description, :type,
-        #                               or any other key in this list.
-        #
-        #   :uniqueItems    Boolean     Whether or not an array should contain only unique items.
-        #
-        #   :properties     Hash        A hash where keys are sub-options and values are definitions
-        #                               for the sub-option. Similar to values for top-level options,
-        #                               properties can have a :description, :type, or any other key
-        #                               in this list.
-        #
-        #   :additionalProperties       A variation of the :properties key, where the hash is a
-        #                   Hash        definition for any properties not specified in :properties.
-        #                               This can be used to permit arbitrary sub-options, such as
-        #                               logs for the 'log' option.
-        #
-        #   :propertyNames  Hash        A hash that defines the properties that an option's property
-        #                               names must adhere to.
-        #
-        #   :required       Array       An array of properties that are required for options that
-        #                               accept Hash values.
-        #
-        #   :minimum        Integer     The minimum integer value for an option.
-        #
-        #   :enum           Array       An array of values that the option recognizes.
-        #
-        #   :pattern        String      A JSON regex pattern that the option's vaue should match.
-        #
-        #   :format         String      Requires that a string value matches a format defined by the
-        #                               JSON Schema draft.
-        #
-        #   :_plugin        Boolean     Whether the option accepts a plugin reference. This is used
-        #                               when generating the JSON schemas to determine whether or not
-        #                               to include a reference to the _plugin definition. If :_plugin
-        #                               is set to true, the script that generates JSON schemas will
-        #                               automatically recurse through the :items and :properties keys
-        #                               and add plugin references if applicable.
-        #
-        #   :_example       Any         An example value for the option. This is used to generate
-        #                               reference documentation for configuration files.
-        #
-        #   :_default       Any         The documented default value for the option. This is only
-        #                               used to generate reference documentation for configuration
-        #                               files and is not used by Bolt to actually set default values.
+        # Definitions used to validate config options.
+        # https://github.com/puppetlabs/bolt/blob/main/schemas/README.md
         TRANSPORT_OPTIONS = {
           "basic-auth-only" => {
             type: [TrueClass, FalseClass],
@@ -81,6 +16,13 @@ module Bolt
             _default: false,
             _example: true
           },
+          "bundled-ruby" => {
+            description: "Whether to use the Ruby bundled with Bolt packages for local targets.",
+            type: [TrueClass, FalseClass],
+            _plugin: false,
+            _example: true,
+            _default: false
+          },
           "cacert" => {
             type: String,
             description: "The path to the CA certificate.",
@@ -201,7 +143,8 @@ module Bolt
                          "`task.py`) and the extension is case sensitive. When a target's name is `localhost`, "\
                          "Ruby tasks run with the Bolt Ruby interpreter by default.",
             additionalProperties: {
-              type: String
+              type: String,
+              _plugin: false
             },
             propertyNames: {
               pattern: "^.?[a-zA-Z0-9]+$"
@@ -357,7 +300,7 @@ module Bolt
             description: "The URL of the host used for API requests.",
             format: "uri",
             _plugin: true,
-            _example: "https://api.example.com:<port>"
+            _example: "https://api.example.com:8143"
           },
           "shell-command" => {
             type: String,

data/lib/bolt/config/transport/ssh.rb

@@ -73,6 +73,14 @@ module Bolt
           %w[ssh-command native-ssh].concat(OPTIONS)
         end
 
+        def self.schema
+          {
+            type:       Hash,
+            properties: self::TRANSPORT_OPTIONS.slice(*(self::OPTIONS + self::NATIVE_OPTIONS)),
+            _plugin:    true
+          }
+        end
+
         private def filter(unfiltered)
           # Because we filter before merging config together it's impossible to
           # know whether both ssh-command *and* native-ssh will be specified
@@ -87,12 +95,6 @@ module Bolt
           super
 
           if (key_opt = @config['private-key'])
-            unless key_opt.instance_of?(String) || (key_opt.instance_of?(Hash) && key_opt.include?('key-data'))
-              raise Bolt::ValidationError,
-                    "private-key option must be a path to a private key file or a Hash containing the 'key-data', "\
-                    "received #{key_opt.class} #{key_opt}"
-            end
-
             if key_opt.instance_of?(String)
               @config['private-key'] = File.expand_path(key_opt, @project)
 
@@ -109,19 +111,6 @@ module Bolt
             @config['interpreters'] = normalize_interpreters(@config['interpreters'])
           end
 
-          if @config['login-shell'] && !LOGIN_SHELLS.include?(@config['login-shell'])
-            raise Bolt::ValidationError,
-                  "Unsupported login-shell #{@config['login-shell']}. Supported shells are #{LOGIN_SHELLS.join(', ')}"
-          end
-
-          %w[encryption-algorithms host-key-algorithms kex-algorithms mac-algorithms run-as-command].each do |opt|
-            next unless @config.key?(opt)
-            unless @config[opt].all? { |n| n.is_a?(String) }
-              raise Bolt::ValidationError,
-                    "#{opt} must be an Array of Strings, received #{@config[opt].inspect}"
-            end
-          end
-
           if @config['login-shell'] == 'powershell'
             %w[tty run-as].each do |key|
               if @config[key]

data/lib/bolt/error.rb

@@ -90,6 +90,20 @@ module Bolt
     end
   end
 
+  class ParallelFailure < Bolt::Error
+    def initialize(results, failed_indices)
+      details = {
+        'action' => 'parallelize',
+        'failed_indices' => failed_indices,
+        'results' => results
+      }
+      message = "Plan aborted: parallel block failed on #{failed_indices.length} target"
+      message += "s" unless failed_indices.length == 1
+      super(message, 'bolt/parallel-failure', details)
+      @error_code = 2
+    end
+  end
+
   class PlanFailure < Error
     def initialize(*args)
       super(*args)
@@ -131,6 +145,16 @@ module Bolt
     end
   end
 
+  class InvalidParallelResult < Error
+    def initialize(result_str, file, line)
+      super("Parallel block returned an invalid result: #{result_str}",
+            'bolt/invalid-plan-result',
+            { 'file' => file,
+              'line' => line,
+              'result_string' => result_str })
+    end
+  end
+
   class ValidationError < Bolt::Error
     def initialize(msg)
       super(msg, 'bolt/validation-error')

data/lib/bolt/executor.rb

@@ -17,6 +17,7 @@ require 'bolt/transport/orch'
 require 'bolt/transport/local'
 require 'bolt/transport/docker'
 require 'bolt/transport/remote'
+require 'bolt/yarn'
 
 module Bolt
   TRANSPORTS = {
@@ -29,7 +30,7 @@ module Bolt
   }.freeze
 
   class Executor
-    attr_reader :noop, :transports
+    attr_reader :noop, :transports, :in_parallel
     attr_accessor :run_as
 
     def initialize(concurrency = 1,
@@ -60,6 +61,7 @@ module Bolt
 
       @noop = noop
       @run_as = nil
+      @in_parallel = false
       @pool = if concurrency > 0
                 Concurrent::ThreadPoolExecutor.new(name: 'exec', max_threads: concurrency)
               else
@@ -84,6 +86,14 @@ module Bolt
       self
     end
 
+    def unsubscribe(subscriber, types = nil)
+      if types.nil? || types.sort == @subscribers[subscriber]&.sort
+        @subscribers.delete(subscriber)
+      elsif @subscribers[subscriber].is_a?(Array)
+        @subscribers[subscriber] = @subscribers[subscriber] - types
+      end
+    end
+
     def publish_event(event)
       @subscribers.each do |subscriber, types|
         # If types isn't set or if the subscriber is subscribed to
@@ -246,10 +256,10 @@ module Bolt
       @logger.trace { "Failed to submit analytics event: #{e.message}" }
     end
 
-    def with_node_logging(description, batch)
-      @logger.info("#{description} on #{batch.map(&:safe_name)}")
+    def with_node_logging(description, batch, log_level = :info)
+      @logger.send(log_level, "#{description} on #{batch.map(&:safe_name)}")
       result = yield
-      @logger.info(result.to_json)
+      @logger.send(log_level, result.to_json)
       result
     end
 
@@ -279,32 +289,20 @@ module Bolt
       end
     end
 
-    def run_task(targets, task, arguments, options = {}, position = [])
+    def run_task(targets, task, arguments, options = {}, position = [], log_level = :info)
       description = options.fetch(:description, "task #{task.name}")
       log_action(description, targets) do
         options[:run_as] = run_as if run_as && !options.key?(:run_as)
         arguments['_task'] = task.name
 
         batch_execute(targets) do |transport, batch|
-          with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch) do
+          with_node_logging("Running task #{task.name} with '#{arguments.to_json}'", batch, log_level) do
             transport.batch_task(batch, task, arguments, options, position, &method(:publish_event))
           end
         end
       end
     end
 
-    def run_task_with_minimal_logging(targets, task, arguments, options = {})
-      description = options.fetch(:description, "task #{task.name}")
-      log_action(description, targets) do
-        options[:run_as] = run_as if run_as && !options.key?(:run_as)
-        arguments['_task'] = task.name
-
-        batch_execute(targets) do |transport, batch|
-          transport.batch_task(batch, task, arguments, options, [], &method(:publish_event))
-        end
-      end
-    end
-
     def run_task_with(target_mapping, task, options = {}, position = [])
       targets = target_mapping.keys
       description = options.fetch(:description, "task #{task.name}")
@@ -359,6 +357,82 @@ module Bolt
       plan.call_by_name_with_scope(scope, params, true)
     end
 
+    def create_yarn(scope, block, object, index)
+      fiber = Fiber.new do
+        # Create the new scope
+        newscope = Puppet::Parser::Scope.new(scope.compiler)
+        local = Puppet::Parser::Scope::LocalScope.new
+
+        # Compress the current scopes into a single vars hash to add to the new scope
+        current_scope = scope.effective_symtable(true)
+        until current_scope.nil?
+          current_scope.instance_variable_get(:@symbols)&.each_pair { |k, v| local[k] = v }
+          current_scope = current_scope.parent
+        end
+        newscope.push_ephemerals([local])
+
+        begin
+          result = catch(:return) do
+            args = { block.parameters[0][1].to_s => object }
+            block.closure.call_by_name_with_scope(newscope, args, true)
+          end
+
+          # If we got a return from the block, get it's value
+          # Otherwise the result is the last line from the block
+          result = result.value if result.is_a?(Puppet::Pops::Evaluator::Return)
+
+          # Validate the result is a PlanResult
+          unless Puppet::Pops::Types::TypeParser.singleton.parse('Boltlib::PlanResult').instance?(result)
+            raise Bolt::InvalidParallelResult.new(result.to_s, *Puppet::Pops::PuppetStack.top_of_stack)
+          end
+
+          result
+        rescue Puppet::PreformattedError => e
+          if e.cause.is_a?(Bolt::Error)
+            e.cause
+          else
+            raise e
+          end
+        end
+      end
+
+      Bolt::Yarn.new(fiber, index)
+    end
+
+    def handle_event(event)
+      case event[:type]
+      when :node_result
+        @thread_completed = true
+      end
+    end
+
+    def round_robin(skein)
+      subscribe(self, [:node_result])
+      results = Array.new(skein.length)
+      @in_parallel = true
+
+      until skein.empty?
+        @thread_completed = false
+        r = nil
+
+        skein.each do |yarn|
+          if yarn.alive?
+            r = yarn.resume
+          else
+            results[yarn.index] = yarn.value
+            skein.delete(yarn)
+          end
+        end
+
+        next unless r == 'unfinished'
+        sleep(0.1) until @thread_completed || skein.empty?
+      end
+
+      @in_parallel = false
+      unsubscribe(self, [:node_result])
+      results
+    end
+
     class TimeoutError < RuntimeError; end
 
     def wait_until_available(targets,

data/lib/bolt/inventory.rb

@@ -4,13 +4,17 @@ require 'set'
 require 'bolt/config'
 require 'bolt/inventory/group'
 require 'bolt/inventory/inventory'
+require 'bolt/inventory/options'
 require 'bolt/target'
 require 'bolt/util'
 require 'bolt/plugin'
+require 'bolt/validator'
 require 'yaml'
 
 module Bolt
   class Inventory
+    include Bolt::Inventory::Options
+
     ENVIRONMENT_VAR = 'BOLT_INVENTORY'
 
     class ValidationError < Bolt::Error
@@ -45,11 +49,25 @@ module Bolt
       end
     end
 
+    # Builds the schema used by the validator.
+    #
+    def self.schema
+      schema = {
+        type:        Hash,
+        properties:  OPTIONS.map { |opt| [opt, _ref: opt] }.to_h,
+        definitions: DEFINITIONS
+      }
+
+      schema[:definitions]['config'][:properties] = Bolt::Config.transport_definitions
+      schema
+    end
+
     def self.from_config(config, plugins)
       logger = Bolt::Logger.logger(self)
 
       if ENV.include?(ENVIRONMENT_VAR)
         begin
+          source = ENVIRONMENT_VAR
           data = YAML.safe_load(ENV[ENVIRONMENT_VAR])
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}" unless data.is_a?(Hash)
           logger.debug("Loaded inventory from environment variable #{ENVIRONMENT_VAR}")
@@ -57,9 +75,11 @@ module Bolt
           raise Bolt::ParseError, "Could not parse inventory from $#{ENVIRONMENT_VAR}"
         end
       elsif config.inventoryfile
+        source = config.inventoryfile
         data = Bolt::Util.read_yaml_hash(config.inventoryfile, 'inventory')
         logger.debug("Loaded inventory from #{config.inventoryfile}")
       else
+        source = config.default_inventoryfile
         data = Bolt::Util.read_optional_yaml_hash(config.default_inventoryfile, 'inventory')
 
         if config.default_inventoryfile.exist?
@@ -74,6 +94,11 @@ module Bolt
         t.resolve(plugins) unless t.resolved?
       end
 
+      Bolt::Validator.new.tap do |validator|
+        validator.validate(data, schema, source)
+        validator.warnings.each { |warning| logger.warn(warning) }
+      end
+
       inventory = create_version(data, config.transport, config.transports, plugins)
       inventory.validate
       inventory