bolt 2.33.1 → 2.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf27b6c8b1a7e3cfb2eb61617bd7ab4927161c577df4915d959182fd8bce7eaf
-  data.tar.gz: '05578a39ce18d1d63bf9455c8890e1e328bdcb3f52dacc3d6d5047251bca833e'
+  metadata.gz: 2cbd90e42181c76bad4eee913ee67369202296852cfa5153a1f5490def4e77d3
+  data.tar.gz: 87ac43d10edeebefaa31f581f49cbcf9f729e25f5d7efec583e09dcf04f97be6
 SHA512:
-  metadata.gz: 11b3fad81f576fa435b570b15ee7cbab14b7c099b4b179770eb20dbc7f2a2db9b2d2de6c4a52eb09021a9794d13c4da704eb3a2ed362cfd50ab1946e7ef5998d
-  data.tar.gz: 300b9fa2214a66d239218b7a8c12518a97c2cf2fafd401c860a3ff9cdd439355608b30d72d5a82f157eea842a25a7c5bc9d484dd9f616837ebfe4beff26c0795
+  metadata.gz: 392f0d453c49a536852692e873e5c5f55807294fa0bd914593722c26d39459e494b83453a3b995319b04b8f2e6f3a1f04f7661400841192bdffc56df47e4170e
+  data.tar.gz: 750cd8e9c29008e8bb4dfc7350d52a268c01d91fa088e9007ff241648e31a90e76d9cabfef9f1db5675d33f6b82b2898a54f758bc21c17c18d888919d0441807

data/Puppetfile

@@ -7,7 +7,7 @@ moduledir File.join(File.dirname(__FILE__), 'modules')
 # Core modules used by 'apply'
 mod 'puppetlabs-service', '1.3.0'
 mod 'puppetlabs-puppet_agent', '4.2.0'
-mod 'puppetlabs-facts', '1.1.0'
+mod 'puppetlabs-facts', '1.2.0'
 
 # Core types and providers for Puppet 6
 mod 'puppetlabs-augeas_core', '1.1.1'

data/bolt-modules/boltlib/lib/puppet/datatypes/applyresult.rb

@@ -12,6 +12,7 @@ Puppet::DataTypes.create_type('ApplyResult') do
       message => Callable[[], Optional[String]],
       action => Callable[[], String],
       to_data => Callable[[], Hash],
+      value => Callable[[], Hash]
     }
   PUPPET
 

data/bolt-modules/boltlib/lib/puppet/functions/catch_errors.rb

@@ -44,9 +44,7 @@ Puppet::Functions.create_function(:catch_errors) do
       yield
     rescue Puppet::PreformattedError => e
       if e.cause.is_a?(Bolt::Error)
-        if error_types.nil?
-          e.cause.to_puppet_error
-        elsif error_types.include?(e.cause.to_h['kind'])
+        if error_types.nil? || error_types.include?(e.cause.to_h['kind'])
           e.cause.to_puppet_error
         else
           raise e

data/bolt-modules/boltlib/lib/puppet/functions/download_file.rb

@@ -110,14 +110,25 @@ Puppet::Functions.create_function(:download_file, Puppet::Functions::InternalFun
     targets = inventory.get_targets(targets)
     if targets.empty?
       call_function('debug', "Simulating file download of '#{source}' - no targets given - no action taken")
-      r = Bolt::ResultSet.new([])
+      Bolt::ResultSet.new([])
     else
-      r = executor.download_file(targets, source, destination, options, Puppet::Pops::PuppetStack.top_of_stack)
-    end
+      r = if executor.in_parallel
+            require 'concurrent'
+            require 'fiber'
+            future = Concurrent::Future.execute do
+              executor.download_file(targets, source, destination, options, Puppet::Pops::PuppetStack.top_of_stack)
+            end
+
+            Fiber.yield('unfinished') while future.incomplete?
+            future.value || future.reason
+          else
+            executor.download_file(targets, source, destination, options, Puppet::Pops::PuppetStack.top_of_stack)
+          end
 
-    if !r.ok && !options[:catch_errors]
-      raise Bolt::RunFailure.new(r, 'download_file', source)
+      if !r.ok && !options[:catch_errors]
+        raise Bolt::RunFailure.new(r, 'download_file', source)
+      end
+      r
     end
-    r
   end
 end

data/bolt-modules/boltlib/lib/puppet/functions/parallelize.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'bolt/yarn'
+
+# Map a code block onto an array, where each array element executes in parallel.
+# This function is experimental.
+#
+# > **Note:** Not available in apply block.
+Puppet::Functions.create_function(:parallelize, Puppet::Functions::InternalFunction) do
+  # Map a block onto an array, where each array element executes in parallel.
+  # This function is experimental.
+  # @param data The array to apply the block to.
+  # @return [Array] An array of PlanResult objects. Each input from the input
+  #   array returns a corresponding PlanResult object.
+  # @example Execute two tasks on multiple targets. Once the task finishes on one
+  #   target, that target can move to the next step without waiting for the task
+  #   to finish on the second target.
+  # $targets = get_targets(["host1", "host2"])
+  # $result = parallelize ($targets) |$t| {
+  #   run_task('a', $t)
+  #   run_task('b', $t)
+  # }
+  dispatch :parallelize do
+    scope_param
+    param 'Array[Any]', :data
+    block_param 'Callable[Any]', :block
+    return_type 'Array[Boltlib::PlanResult]'
+  end
+
+  def parallelize(scope, data, &block)
+    unless Puppet[:tasks]
+      raise Puppet::ParseErrorWithIssue
+        .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'parallelize')
+    end
+
+    executor = Puppet.lookup(:bolt_executor)
+    executor.report_function_call(self.class.name)
+
+    skein = data.each_with_index.map do |object, index|
+      executor.create_yarn(scope, block, object, index)
+    end
+
+    result = executor.round_robin(skein)
+
+    failed_indices = result.each_index.select do |i|
+      result[i].is_a?(Bolt::Error)
+    end
+
+    # TODO: Inner catch errors block?
+    if failed_indices.any?
+      raise Bolt::ParallelFailure.new(result, failed_indices)
+    end
+
+    result
+  end
+end

data/bolt-modules/boltlib/lib/puppet/functions/run_command.rb

@@ -67,14 +67,32 @@ Puppet::Functions.create_function(:run_command) do
 
     if targets.empty?
       call_function('debug', "Simulating run_command('#{command}') - no targets given - no action taken")
-      r = Bolt::ResultSet.new([])
+      Bolt::ResultSet.new([])
     else
-      r = executor.run_command(targets, command, options, Puppet::Pops::PuppetStack.top_of_stack)
-    end
+      r = if executor.in_parallel
+            require 'concurrent'
+            require 'fiber'
+            future = Concurrent::Future.execute do
+              executor.run_command(targets,
+                                   command,
+                                   options,
+                                   Puppet::Pops::PuppetStack.top_of_stack)
+            end
+
+            Fiber.yield('unfinished') while future.incomplete?
+            future.value || future.reason
+          else
+            executor.run_command(targets,
+                                 command,
+                                 options,
+                                 Puppet::Pops::PuppetStack.top_of_stack)
+          end
+
+      if !r.ok && !options[:catch_errors]
+        raise Bolt::RunFailure.new(r, 'run_command', command)
+      end
 
-    if !r.ok && !options[:catch_errors]
-      raise Bolt::RunFailure.new(r, 'run_command', command)
+      r
     end
-    r
   end
 end

data/bolt-modules/boltlib/lib/puppet/functions/run_script.rb

@@ -84,15 +84,34 @@ Puppet::Functions.create_function(:run_script, Puppet::Functions::InternalFuncti
     # Ensure that given targets are all Target instances)
     targets = inventory.get_targets(targets)
 
-    r = if targets.empty?
-          Bolt::ResultSet.new([])
-        else
-          executor.run_script(targets, found, arguments, options, Puppet::Pops::PuppetStack.top_of_stack)
-        end
+    if targets.empty?
+      Bolt::ResultSet.new([])
+    else
+      r = if executor.in_parallel
+            require 'concurrent'
+            require 'fiber'
+            future = Concurrent::Future.execute do
+              executor.run_script(targets,
+                                  found,
+                                  arguments,
+                                  options,
+                                  Puppet::Pops::PuppetStack.top_of_stack)
+            end
 
-    if !r.ok && !options[:catch_errors]
-      raise Bolt::RunFailure.new(r, 'run_script', script)
+            Fiber.yield('unfinished') while future.incomplete?
+            future.value || future.reason
+          else
+            executor.run_script(targets,
+                                found,
+                                arguments,
+                                options,
+                                Puppet::Pops::PuppetStack.top_of_stack)
+          end
+
+      if !r.ok && !options[:catch_errors]
+        raise Bolt::RunFailure.new(r, 'run_script', script)
+      end
+      r
     end
-    r
   end
 end

data/bolt-modules/boltlib/lib/puppet/functions/run_task.rb

@@ -133,7 +133,27 @@ Puppet::Functions.create_function(:run_task) do
     if targets.empty?
       Bolt::ResultSet.new([])
     else
-      result = executor.run_task(targets, task, params, options, Puppet::Pops::PuppetStack.top_of_stack)
+      result = if executor.in_parallel
+                 require 'concurrent'
+                 require 'fiber'
+                 future = Concurrent::Future.execute do
+                   executor.run_task(targets,
+                                     task,
+                                     params,
+                                     options,
+                                     Puppet::Pops::PuppetStack.top_of_stack)
+                 end
+
+                 Fiber.yield('unfinished') while future.incomplete?
+                 future.value || future.reason
+               else
+                 executor.run_task(targets,
+                                   task,
+                                   params,
+                                   options,
+                                   Puppet::Pops::PuppetStack.top_of_stack)
+               end
+
       if !result.ok && !options[:catch_errors]
         raise Bolt::RunFailure.new(result, 'run_task', task_name)
       end

data/bolt-modules/boltlib/lib/puppet/functions/run_task_with.rb

@@ -180,7 +180,24 @@ Puppet::Functions.create_function(:run_task_with) do
     else
       # Combine the results from the task run with any failing results that were
       # generated earlier when creating the target mapping
-      task_result = executor.run_task_with(target_mapping, task, options, Puppet::Pops::PuppetStack.top_of_stack)
+      task_result = if executor.in_parallel
+                      require 'concurrent'
+                      require 'fiber'
+                      future = Concurrent::Future.execute do
+                        executor.run_task_with(target_mapping,
+                                               task,
+                                               options,
+                                               Puppet::Pops::PuppetStack.top_of_stack)
+                      end
+
+                      Fiber.yield('unfinished') while future.incomplete?
+                      future.value || future.reason
+                    else
+                      executor.run_task_with(target_mapping,
+                                             task,
+                                             options,
+                                             Puppet::Pops::PuppetStack.top_of_stack)
+                    end
       result = Bolt::ResultSet.new(task_result.results + error_set)
 
       if !result.ok && !options[:catch_errors]

data/bolt-modules/boltlib/lib/puppet/functions/upload_file.rb

@@ -81,14 +81,32 @@ Puppet::Functions.create_function(:upload_file, Puppet::Functions::InternalFunct
     targets = inventory.get_targets(targets)
     if targets.empty?
       call_function('debug', "Simulating file upload of '#{found}' - no targets given - no action taken")
-      r = Bolt::ResultSet.new([])
+      Bolt::ResultSet.new([])
     else
-      r = executor.upload_file(targets, found, destination, options, Puppet::Pops::PuppetStack.top_of_stack)
-    end
+      r = if executor.in_parallel
+            require 'concurrent'
+            require 'fiber'
+            future = Concurrent::Future.execute do
+              executor.upload_file(targets,
+                                   found,
+                                   destination,
+                                   options,
+                                   Puppet::Pops::PuppetStack.top_of_stack)
+            end
 
-    if !r.ok && !options[:catch_errors]
-      raise Bolt::RunFailure.new(r, 'upload_file', source)
+            Fiber.yield('unfinished') while future.incomplete?
+            future.value || future.reason
+          else
+            executor.upload_file(targets,
+                                 found,
+                                 destination,
+                                 options,
+                                 Puppet::Pops::PuppetStack.top_of_stack)
+          end
+      if !r.ok && !options[:catch_errors]
+        raise Bolt::RunFailure.new(r, 'upload_file', source)
+      end
+      r
     end
-    r
   end
 end

data/lib/bolt/analytics.rb

@@ -2,6 +2,7 @@
 
 require 'bolt/util'
 require 'bolt/version'
+require 'find'
 require 'json'
 require 'logging'
 require 'securerandom'
@@ -23,14 +24,16 @@ module Bolt
       plan_steps: :cd8,
       return_type: :cd9,
       inventory_version: :cd10,
-      boltdir_type: :cd11
+      boltdir_type: :cd11,
+      puppet_plan_count: :cd12,
+      yaml_plan_count: :cd13
     }.freeze
 
     def self.build_client
       logger = Bolt::Logger.logger(self)
       begin
-        config_file = config_path(logger)
-        config = load_config(config_file, logger)
+        config_file = config_path
+        config = load_config(config_file)
       rescue ArgumentError
         config = { 'disabled' => true }
       end
@@ -51,7 +54,7 @@ module Bolt
       NoopClient.new
     end
 
-    def self.config_path(logger)
+    def self.config_path
       path     = File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt', 'analytics.yaml'))
       old_path = File.expand_path(File.join('~', '.puppetlabs', 'bolt', 'analytics.yaml'))
 
@@ -59,7 +62,7 @@ module Bolt
         if File.exist?(old_path)
           message = "Detected analytics configuration files at '#{old_path}' and '#{path}'. Loading "\
                     "analytics configuration from '#{path}'."
-          logger.warn(message)
+          Bolt::Logger.warn_once('duplicate_analytics', message)
         end
 
         path
@@ -70,12 +73,12 @@ module Bolt
       end
     end
 
-    def self.load_config(filename, logger)
+    def self.load_config(filename)
       if File.exist?(filename)
         Bolt::Util.read_optional_yaml_hash(filename, 'analytics')
       else
         unless ENV['BOLT_DISABLE_ANALYTICS']
-          logger.warn <<~ANALYTICS
+          Bolt::Logger.warn_once('analytics_opt_out', <<~ANALYTICS)
             Bolt collects data about how you use it. You can opt out of providing this data.
 
             To disable analytics data collection, add this line to ~/.puppetlabs/etc/bolt/analytics.yaml :
@@ -134,11 +137,23 @@ module Bolt
       end
 
       def report_bundled_content(mode, name)
-        if bundled_content[mode.split(' ').first]&.include?(name)
+        if bundled_content[mode.split.first]&.include?(name)
           event('Bundled Content', mode, label: name)
         end
       end
 
+      def plan_counts(plans_path)
+        pp_count, yaml_count = if File.exist?(plans_path)
+                                 %w[pp yaml].map do |extension|
+                                   Find.find(plans_path.to_s).grep(/.*\.#{extension}/).length
+                                 end
+                               else
+                                 [0, 0]
+                               end
+
+        { puppet_plan_count: pp_count, yaml_plan_count: yaml_count }
+      end
+
       def event(category, action, label: nil, value: nil, **kwargs)
         custom_dimensions = Bolt::Util.walk_keys(kwargs) do |k|
           CUSTOM_DIMENSIONS[k] || raise("Unknown analytics key '#{k}'")
@@ -224,6 +239,10 @@ module Bolt
 
       def report_bundled_content(mode, name); end
 
+      def plan_counts(_)
+        {}
+      end
+
       def event(category, action, **_kwargs)
         @logger.trace "Skipping submission of '#{category} #{action}' event because analytics is disabled"
       end

data/lib/bolt/apply_result.rb

@@ -96,9 +96,9 @@ module Bolt
       @target = target
       @value = {}
       @action = 'apply'
-      value['report'] = report if report
-      value['_error'] = error if error
-      value['_output'] = metrics_message if metrics_message
+      @value['report'] = report if report
+      @value['_error'] = error if error
+      @value['_output'] = metrics_message if metrics_message
     end
 
     def event_metrics

data/lib/bolt/bolt_option_parser.rb

@@ -6,14 +6,15 @@ require 'optparse'
 
 module Bolt
   class BoltOptionParser < OptionParser
+    PROJECT_PATHS = %w[project configfile boltdir].freeze
     OPTIONS = { inventory: %w[targets query rerun description],
                 authentication: %w[user password password-prompt private-key host-key-check ssl ssl-verify],
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
-                global_config_setters: %w[modulepath project configfile],
+                global_config_setters: PROJECT_PATHS + %w[modulepath],
                 transports: %w[transport connect-timeout tty native-ssh ssh-command copy-command],
                 display: %w[format color verbose trace],
-                global: %w[help version debug log-level] }.freeze
+                global: %w[help version debug log-level clear-cache] }.freeze
 
     ACTION_OPTS = OPTIONS.values.flatten.freeze
 
@@ -46,7 +47,8 @@ module Bolt
       when 'inventory'
         case action
         when 'show'
-          { flags: OPTIONS[:inventory] + OPTIONS[:global] + %w[format inventoryfile boltdir configfile detail],
+          { flags: OPTIONS[:inventory] + OPTIONS[:global] +
+            PROJECT_PATHS + %w[format inventoryfile detail],
             banner: INVENTORY_SHOW_HELP }
         else
           { flags: OPTIONS[:global],
@@ -55,7 +57,7 @@ module Bolt
       when 'group'
         case action
         when 'show'
-          { flags: OPTIONS[:global] + %w[format inventoryfile boltdir configfile],
+          { flags: OPTIONS[:global] + PROJECT_PATHS + %w[format inventoryfile],
             banner: GROUP_SHOW_HELP }
         else
           { flags: OPTIONS[:global],
@@ -67,13 +69,13 @@ module Bolt
       when 'module'
         case action
         when 'add'
-          { flags: OPTIONS[:global] + %w[configfile project],
+          { flags: OPTIONS[:global] + PROJECT_PATHS,
             banner: MODULE_ADD_HELP }
         when 'generate-types'
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters],
             banner: MODULE_GENERATETYPES_HELP }
         when 'install'
-          { flags: OPTIONS[:global] + %w[configfile force project resolve],
+          { flags: OPTIONS[:global] + PROJECT_PATHS + %w[force resolve],
             banner: MODULE_INSTALL_HELP }
         when 'show'
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters],
@@ -88,7 +90,7 @@ module Bolt
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters],
             banner: PLAN_CONVERT_HELP }
         when 'new'
-          { flags: OPTIONS[:global] + %w[configfile project],
+          { flags: OPTIONS[:global] + PROJECT_PATHS + %w[pp],
             banner: PLAN_NEW_HELP }
         when 'run'
           { flags: ACTION_OPTS + %w[params compile-concurrency tmpdir hiera-config],
@@ -106,7 +108,7 @@ module Bolt
           { flags: OPTIONS[:global] + %w[modules],
             banner: PROJECT_INIT_HELP }
         when 'migrate'
-          { flags: OPTIONS[:global] + %w[inventoryfile project configfile],
+          { flags: OPTIONS[:global] + PROJECT_PATHS + %w[inventoryfile],
             banner: PROJECT_MIGRATE_HELP }
         else
           { flags: OPTIONS[:global],
@@ -793,7 +795,10 @@ module Bolt
         @options[:noop] = true
       end
       define('--description DESCRIPTION',
-             'Description to use for the job') do |description|
+             'Deprecated. Description to use for the job') do |description|
+        msg = "Command line option '--description' is deprecated, and will be "\
+          "removed in Bolt 3.0."
+        @deprecations << { type: 'Using --description', msg: msg }
         @options[:description] = description
       end
       define('--params PARAMETERS',
@@ -873,13 +878,25 @@ module Bolt
           File.expand_path(moduledir)
         end
       end
-      define('--project PATH', '--boltdir PATH',
-             'Specify what project to load config from (default: autodiscovered from current working dir)') do |path|
+      define('--boltdir PATH',
+             'Deprecated. Specify what project to load config from (default:',
+             'autodiscovered from current working dir)') do |path|
+        msg = "Command line option '--boltdir' is deprecated, use '--project' instead."
+        @deprecations << { type: 'Using --boltdir', msg: msg }
         @options[:boltdir] = path
       end
+      define('--project PATH',
+             'Path to load the Bolt project from (default: autodiscovered from current dir)') do |path|
+        @options[:project] = path
+      end
       define('--configfile PATH',
-             'Specify where to load config from (default: ~/.puppetlabs/bolt/bolt.yaml).',
-             'Directory containing bolt.yaml will be used as the project directory.') do |path|
+             'Deprecated. Specify where to load config from (default:',
+             '~/.puppetlabs/bolt/bolt.yaml). Directory containing bolt.yaml will be',
+             'used as the project directory.') do |path|
+        msg = "Command line option '--configfile' is deprecated, and " \
+          "will be removed in Bolt 3.0. Use '--project' and provide the "\
+          "directory path instead."
+        @deprecations << { type: 'Using --configfile', msg: msg }
         @options[:configfile] = path
       end
       define('--hiera-config PATH',
@@ -891,12 +908,18 @@ module Bolt
         if ENV.include?(Bolt::Inventory::ENVIRONMENT_VAR)
           raise Bolt::CLIError, "Cannot pass inventory file when #{Bolt::Inventory::ENVIRONMENT_VAR} is set"
         end
-        @options[:inventoryfile] = Pathname.new(File.expand_path(path))
+        @options[:inventoryfile] = File.expand_path(path)
       end
       define('--puppetfile PATH',
-             'Specify a Puppetfile to use when installing modules. (default: ~/.puppetlabs/bolt/Puppetfile)',
+             'Deprecated. Specify a Puppetfile to use when installing modules.',
+             ' (default: ~/.puppetlabs/bolt/Puppetfile)',
              'Modules are installed in the current project.') do |path|
-        @options[:puppetfile_path] = Pathname.new(File.expand_path(path))
+        command = Bolt::Util.powershell? ? 'Update-BoltProject' : 'bolt project migrate'
+        msg = "Command line option '--puppetfile' is deprecated, and will be removed "\
+          "in Bolt 3.0. You can migrate to using the new module management "\
+          "workflow using '#{command}'."
+        @deprecations << { type: 'Using --puppetfile', msg: msg }
+        @options[:puppetfile_path] = File.expand_path(path)
       end
       define('--[no-]save-rerun', 'Whether to update the rerun file after this command.') do |save|
         @options[:'save-rerun'] = save
@@ -945,6 +968,11 @@ module Bolt
         @options[:resolve] = resolve
       end
 
+      separator "\nPLAN OPTIONS"
+      define('--pp', 'Create a new Puppet language plan.') do |_|
+        @options[:puppet] = true
+      end
+
       separator "\nDISPLAY OPTIONS"
       define('--filter FILTER', 'Filter tasks and plans by a matching substring') do |filter|
         unless /^[a-z0-9_:]+$/.match(filter)
@@ -997,6 +1025,10 @@ module Bolt
              "trace, debug, info, warn, error, fatal, any.") do |level|
         @options[:log] = { 'console' => { 'level' => level } }
       end
+      define('--clear-cache',
+             "Clear plugin cache before executing") do |_|
+        @options[:clear_cache] = true
+      end
       define('--plugin PLUGIN', 'Select the plugin to use') do |plug|
         @options[:plugin] = plug
       end