bolt 2.17.0 → 2.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d7b3fbf02e3b0162ead0c7ca8994eb6a96f5f8d49d9b27aa4620a54c111396d
-  data.tar.gz: 3ec22c605ca39808ffdc1f9589bdc05921889f4cdf7ef7079682614bd1298a81
+  metadata.gz: 60e995704052fee2778474c532df844e51ca0888017e28c883866e6b61a9678f
+  data.tar.gz: 8133b33137d67ba8880270f4ed20aa8346e13b2aa7ae2b0c390e3fbbba660172
 SHA512:
-  metadata.gz: 71c5644b0b1bcaee1aaf5d74b6e1ae895e82ffa56dc52df9b4fc2995f1fb7541c9da8c3a0a4d31b63b72e04ab2245d4fd377f3bf7eaf18ea4de7f8c7c2f46c6b
-  data.tar.gz: 503755fc3f2196d53fc26d6bfbe662671689ed6208ccf9ab06a384eb254d878f2e00d62b182f3c2185d09fc558bed8a2dfa8d2b942024bd62977b0e50e8c0d33
+  metadata.gz: a057aebe5bd7ce01a8369f3c7a05371f0dab79143cd465f6bd2f0a0ad23fe897fcff6624d03f07ead07b86c888512166232182c294cd1188fcd8fb45b55f463d
+  data.tar.gz: 4c30348d86636398f25568dcd77af25e6eb8e54af5c9ce44125a110b8089c15415a4a051aa8aac066f7404f9ca7faf253dd69e350780ef8d24e34f385c1dd8de

data/Puppetfile

@@ -5,7 +5,7 @@ forge "http://forge.puppetlabs.com"
 moduledir File.join(File.dirname(__FILE__), 'modules')
 
 # Core modules used by 'apply'
-mod 'puppetlabs-service', '1.2.0'
+mod 'puppetlabs-service', '1.3.0'
 mod 'puppetlabs-puppet_agent', '3.2.0'
 mod 'puppetlabs-facts', '1.0.0'
 
@@ -28,6 +28,7 @@ mod 'puppetlabs-python_task_helper', '0.4.3'
 mod 'puppetlabs-reboot', '3.0.0'
 mod 'puppetlabs-ruby_task_helper', '0.5.1'
 mod 'puppetlabs-ruby_plugin_helper', '0.1.0'
+mod 'puppetlabs-stdlib', '6.3.0'
 
 # Plugin modules
 mod 'puppetlabs-aws_inventory', '0.5.0'
@@ -42,3 +43,4 @@ mod 'puppetlabs-yaml', '0.2.0'
 mod 'canary', local: true
 mod 'aggregate', local: true
 mod 'puppetdb_fact', local: true
+mod 'secure_env_vars', local: true

data/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb

@@ -13,10 +13,13 @@ require 'bolt/task'
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:apply_prep) do
   # @param targets A pattern or array of patterns identifying a set of targets.
+  # @param options Options hash.
+  # @option options [Array] _required_modules An array of modules to sync to the target.
   # @example Prepare targets by name.
   #   apply_prep('target1,target2')
   dispatch :apply_prep do
     param 'Boltlib::TargetSpec', :targets
+    optional_param 'Hash[String, Data]', :options
   end
 
   def script_compiler
@@ -60,18 +63,34 @@ Puppet::Functions.create_function(:apply_prep) do
     @executor ||= Puppet.lookup(:bolt_executor)
   end
 
-  def apply_prep(target_spec)
+  def apply_prep(target_spec, options = {})
     unless Puppet[:tasks]
       raise Puppet::ParseErrorWithIssue
         .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep')
     end
 
+    options = options.transform_keys { |k| k.sub(/^_/, '').to_sym }
+
     applicator = Puppet.lookup(:apply_executor)
 
     executor.report_function_call(self.class.name)
 
     targets = inventory.get_targets(target_spec)
 
+    required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+    if required_modules&.any?
+      Puppet.debug("Syncing only required modules: #{required_modules.join(',')}.")
+    end
+
+    # Gather facts, including custom facts
+    plugins = applicator.build_plugin_tarball do |mod|
+      next unless required_modules.nil? || required_modules.include?(mod.name)
+      search_dirs = []
+      search_dirs << mod.plugins if mod.plugins?
+      search_dirs << mod.pluginfacts if mod.pluginfacts?
+      search_dirs
+    end
+
     executor.log_action('install puppet and gather facts', targets) do
       executor.without_default_logging do
         # Skip targets that include the puppet-agent feature, as we know an agent will be available.
@@ -109,14 +128,6 @@ Puppet::Functions.create_function(:apply_prep) do
           need_install_targets.each { |target| set_agent_feature(target) }
         end
 
-        # Gather facts, including custom facts
-        plugins = applicator.build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs
-        end
-
         task = applicator.custom_facts_task
         arguments = { 'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins) }
         results = executor.run_task(targets, task, arguments)

data/bolt-modules/boltlib/lib/puppet/functions/download_file.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'bolt/error'
+
+# Downloads the given file or directory from the given set of targets and saves it to a directory
+# matching the target's name under the given destination directory. Returns the result from each
+# download. This does nothing if the list of targets is empty.
+#
+# > **Note:** Existing content in the destination directory is deleted before downloading from
+# > the targets.
+#
+# > **Note:** Not available in apply block
+Puppet::Functions.create_function(:download_file, Puppet::Functions::InternalFunction) do
+  # Download a file or directory.
+  # @param source The absolute path to the file or directory on the target(s).
+  # @param destination The relative path to the destination directory on the local system. Expands
+  #                    relative to `<project>/downloads/`.
+  # @param targets A pattern identifying zero or more targets. See {get_targets} for accepted patterns.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @option options [String] _run_as User to run as using privilege escalation.
+  # @return A list of results, one entry per target, with the path to the downloaded file under the
+  #         `path` key.
+  # @example Download a file from multiple Linux targets to a destination directory
+  #   download_file('/etc/ssh/ssh_config', '~/Downloads', $targets)
+  # @example Download a directory from multiple Linux targets to a project downloads directory
+  #   download_file('/etc/ssh', 'ssh', $targets)
+  # @example Download a file from multiple Linux targets and compare its contents to a local file
+  #   $results = download_file($source, $destination, $targets)
+  #
+  #   $local_content = file::read($source)
+  #
+  #   $mismatched_files = $results.filter |$result| {
+  #     $remote_content = file::read($result['path'])
+  #     $remote_content == $local_content
+  #   }
+  dispatch :download_file do
+    param 'String[1]', :source
+    param 'String[1]', :destination
+    param 'Boltlib::TargetSpec', :targets
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'ResultSet'
+  end
+
+  # Download a file or directory, logging the provided description.
+  # @param source The absolute path to the file or directory on the target(s).
+  # @param destination The relative path to the destination directory on the local system. Expands
+  #                    relative to `<project>/downloads/`.
+  # @param targets A pattern identifying zero or more targets. See {get_targets} for accepted patterns.
+  # @param description A description to be output when calling this function.
+  # @param options A hash of additional options.
+  # @option options [Boolean] _catch_errors Whether to catch raised errors.
+  # @option options [String] _run_as User to run as using privilege escalation.
+  # @return A list of results, one entry per target, with the path to the downloaded file under the
+  #         `path` key.
+  # @example Download a file from multiple Linux targets to a destination directory
+  #   download_file('/etc/ssh/ssh_config', '~/Downloads', $targets, 'Downloading remote SSH config')
+  dispatch :download_file_with_description do
+    param 'String[1]', :source
+    param 'String[1]', :destination
+    param 'Boltlib::TargetSpec', :targets
+    param 'String', :description
+    optional_param 'Hash[String[1], Any]', :options
+    return_type 'ResultSet'
+  end
+
+  def download_file(source, destination, targets, options = {})
+    download_file_with_description(source, destination, targets, nil, options)
+  end
+
+  def download_file_with_description(source, destination, targets, description = nil, options = {})
+    unless Puppet[:tasks]
+      raise Puppet::ParseErrorWithIssue
+        .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'download_file')
+    end
+
+    options = options.select { |opt| opt.start_with?('_') }.transform_keys { |k| k.sub(/^_/, '').to_sym }
+    options[:description] = description if description
+
+    executor = Puppet.lookup(:bolt_executor)
+    inventory = Puppet.lookup(:bolt_inventory)
+
+    if (destination = destination.strip).empty?
+      raise Bolt::ValidationError, "Destination cannot be an empty string"
+    end
+
+    if (destination = Pathname.new(destination)).absolute?
+      raise Bolt::ValidationError, "Destination must be a relative path, received absolute path #{destination}"
+    end
+
+    # Prevent path traversal so downloads can't be saved outside of the project downloads directory
+    if (destination.each_filename.to_a & %w[. ..]).any?
+      raise Bolt::ValidationError, "Destination must not include path traversal, received #{destination}"
+    end
+
+    # Paths expand relative to the default downloads directory for the project
+    # e.g. ~/.puppetlabs/bolt/downloads/
+    destination = Puppet.lookup(:bolt_project_data).downloads + destination
+
+    # If the destination directory already exists, delete any existing contents
+    if Dir.exist?(destination)
+      FileUtils.rm_r(Dir.glob(destination + '*'), secure: true)
+    end
+
+    # Send Analytics Report
+    executor.report_function_call(self.class.name)
+
+    # Ensure that that given targets are all Target instances
+    targets = inventory.get_targets(targets)
+    if targets.empty?
+      call_function('debug', "Simulating file download of '#{source}' - no targets given - no action taken")
+      r = Bolt::ResultSet.new([])
+    else
+      r = executor.download_file(targets, source, destination, options)
+    end
+
+    if !r.ok && !options[:catch_errors]
+      raise Bolt::RunFailure.new(r, 'download_file', source)
+    end
+    r
+  end
+end

data/bolt-modules/boltlib/lib/puppet/functions/run_plan.rb

@@ -11,6 +11,9 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
   # @param args A hash of arguments to the plan. Can also include additional options.
   # @option args [Boolean] _catch_errors Whether to catch raised errors.
   # @option args [String] _run_as User to run as using privilege escalation.
+  #   This option sets the [run-as user](privilege_escalation.md) for all
+  #   targets whenever Bolt connects to a target. This is set for all functions
+  #   in the called plan, including `run_plan()`.
   # @return [PlanResult] The result of running the plan. Undef if plan does not explicitly return results.
   # @example Run a plan
   #   run_plan('canary', 'command' => 'false', 'targets' => $targets, '_catch_errors' => true)
@@ -31,6 +34,9 @@ Puppet::Functions.create_function(:run_plan, Puppet::Functions::InternalFunction
   # @param args A hash of arguments to the plan. Can also include additional options.
   # @option args [Boolean] _catch_errors Whether to catch raised errors.
   # @option args [String] _run_as User to run as using privilege escalation.
+  #   This option sets the [run-as user](privilege_escalation.md) for all
+  #   targets whenever Bolt connects to a target. This is set for all functions
+  #   in the called plan, including `run_plan()`.
   # @return [PlanResult] The result of running the plan. Undef if plan does not explicitly return results.
   # @example Run a plan
   #   run_plan('canary', $targets, 'command' => 'false')

data/bolt-modules/dir/lib/puppet/functions/dir/children.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+# Returns an array containing all of the filenames except for "." and ".." in the given directory.
+Puppet::Functions.create_function(:'dir::children', Puppet::Functions::InternalFunction) do
+  # @param dirname Absolute path or Puppet module name.
+  # @return Array of files in the given directory.
+  # @example List filenames from an absolute path.
+  #   dir::children('/home/user/subdir/')
+  # @example List filenames from a Puppet file path.
+  #   dir::children('puppet_agent')
+  dispatch :children do
+    scope_param
+    required_param 'String', :dirname
+    return_type 'Array'
+  end
+
+  def children(scope, dirname)
+    # Send Analytics Report
+    Puppet.lookup(:bolt_executor) {}&.report_function_call(self.class.name)
+    modname, subpath = dirname.split(File::SEPARATOR, 2)
+    mod_path = scope.compiler.environment.module(modname)&.path
+
+    full_mod_path = File.join(mod_path, subpath || '') if mod_path
+
+    # Expand relative to the project directory if path is relative
+    project = Puppet.lookup(:bolt_project_data)
+    pathname = Pathname.new(dirname)
+    full_dir = pathname.absolute? ? dirname : File.expand_path(File.join(project.path, dirname))
+
+    # Sort for testability
+    Dir.children(full_mod_path || full_dir).sort
+  end
+end

data/lib/bolt/applicator.rb

@@ -18,7 +18,6 @@ module Bolt
                    pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
-
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath || []
@@ -30,17 +29,6 @@ module Bolt
 
       @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles)
       @logger = Logging.logger[self]
-      @plugin_tarball = Concurrent::Delay.new do
-        build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs << mod.files if mod.files?
-          type_files = "#{mod.path}/types"
-          search_dirs << type_files if File.exist?(type_files)
-          search_dirs
-        end
-      end
     end
 
     private def libexec
@@ -188,7 +176,6 @@ module Bolt
 
     def apply_ast(raw_ast, targets, options, plan_vars = {})
       ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
-
       # Serialize as pcore for *Result* objects
       plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
                                                                        rich_data: true,
@@ -207,9 +194,26 @@ module Bolt
         # This data isn't available on the target config hash
         config: @inventory.transport_data_get
       }
-
       description = options[:description] || 'apply catalog'
 
+      required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+      if required_modules&.any?
+        @logger.debug("Syncing only required modules: #{required_modules.join(',')}.")
+      end
+
+      @plugin_tarball = Concurrent::Delay.new do
+        build_plugin_tarball do |mod|
+          next unless required_modules.nil? || required_modules.include?(mod.name)
+          search_dirs = []
+          search_dirs << mod.plugins if mod.plugins?
+          search_dirs << mod.pluginfacts if mod.pluginfacts?
+          search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
+          search_dirs
+        end
+      end
+
       r = @executor.log_action(description, targets) do
         futures = targets.map do |target|
           Concurrent::Future.execute(executor: @pool) do
@@ -236,6 +240,7 @@ module Bolt
                   result
                 end
               else
+
                 arguments = {
                   'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
                   'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),

data/lib/bolt/apply_result.rb

@@ -20,7 +20,7 @@ module Bolt
              error_hash['msg'] =~ /The term 'ruby.exe' is not recognized as the name of a cmdlet/)
         # Windows does not have Ruby present
         {
-          'msg' => "Puppet is not installed on the target in $env:ProgramFiles, please install it to enable 'apply'",
+          'msg' => "Puppet was not found on the target or in $env:ProgramFiles, please install it to enable 'apply'",
           'kind' => 'bolt/apply-error'
         }
       elsif exit_code == 1 && error_hash['msg'] =~ /cannot load such file -- puppet \(LoadError\)/

data/lib/bolt/bolt_option_parser.rb

@@ -11,7 +11,7 @@ module Bolt
                 escalation: %w[run-as sudo-password sudo-password-prompt sudo-executable],
                 run_context: %w[concurrency inventoryfile save-rerun cleanup],
                 global_config_setters: %w[modulepath project configfile],
-                transports: %w[transport connect-timeout tty ssh-command copy-command],
+                transports: %w[transport connect-timeout tty native-ssh ssh-command copy-command],
                 display: %w[format color verbose trace],
                 global: %w[help version debug log-level] }.freeze
 
@@ -25,7 +25,7 @@ module Bolt
       when 'command'
         case action
         when 'run'
-          { flags: ACTION_OPTS,
+          { flags: ACTION_OPTS + %w[env-var],
             banner: COMMAND_RUN_HELP }
         else
           { flags: OPTIONS[:global],
@@ -36,6 +36,9 @@ module Bolt
         when 'upload'
           { flags: ACTION_OPTS + %w[tmpdir],
             banner: FILE_UPLOAD_HELP }
+        when 'download'
+          { flags: ACTION_OPTS,
+            banner: FILE_DOWNLOAD_HELP }
         else
           { flags: OPTIONS[:global],
             banner: FILE_HELP }
@@ -63,6 +66,9 @@ module Bolt
         when 'convert'
           { flags: OPTIONS[:global] + OPTIONS[:global_config_setters],
             banner: PLAN_CONVERT_HELP }
+        when 'new'
+          { flags: OPTIONS[:global] + %w[configfile project],
+            banner: PLAN_NEW_HELP }
         when 'run'
           { flags: ACTION_OPTS + %w[params compile-concurrency tmpdir hiera-config],
             banner: PLAN_RUN_HELP }
@@ -103,7 +109,7 @@ module Bolt
       when 'script'
         case action
         when 'run'
-          { flags: ACTION_OPTS + %w[tmpdir],
+          { flags: ACTION_OPTS + %w[tmpdir env-var],
             banner: SCRIPT_RUN_HELP }
         else
           { flags: OPTIONS[:global],
@@ -156,10 +162,10 @@ module Bolt
       SUBCOMMANDS
           apply             Apply Puppet manifest code
           command           Run a command remotely
-          file              Upload a local file or directory
+          file              Copy files between the controller and targets
           group             Show the list of groups in the inventory
           inventory         Show the list of targets an action would run on
-          plan              Convert, show, and run Bolt plans
+          plan              Convert, create, show, and run Bolt plans
           project           Create and migrate Bolt projects
           puppetfile        Install and list modules and generate type references
           script            Upload a local script and run it remotely
@@ -218,10 +224,30 @@ module Bolt
           bolt file <action> [options]
 
       DESCRIPTION
-          Upload a local file or directory
+          Copy files and directories between the controller and targets
 
       ACTIONS
-          upload        Upload a local file or directory
+          download      Download a file or directory to the controller
+          upload        Upload a local file or directory from the controller
+    HELP
+
+    FILE_DOWNLOAD_HELP = <<~HELP
+      NAME
+          download
+
+      USAGE
+          bolt file download <src> <dest> [options]
+
+      DESCRIPTION
+          Download a file or directory from one or more targets.
+
+          Downloaded files and directories are saved to the a subdirectory
+          matching the target's name under the destination directory. The
+          destination directory is expanded relative to the downloads
+          subdirectory of the project directory.
+
+      EXAMPLES
+          bolt file download /etc/ssh_config ssh_config -t all
     HELP
 
     FILE_UPLOAD_HELP = <<~HELP
@@ -296,10 +322,11 @@ module Bolt
           bolt plan <action> [parameters] [options]
 
       DESCRIPTION
-          Convert, show, and run Bolt plans.
+          Convert, create, show, and run Bolt plans.
 
       ACTIONS
           convert       Convert a YAML plan to a Bolt plan
+          new           Create a new plan in the current project
           run           Run a plan on the specified targets
           show          Show available plans and plan documentation
     HELP
@@ -322,6 +349,20 @@ module Bolt
           bolt plan convert path/to/plan/myplan.yaml
     HELP
 
+    PLAN_NEW_HELP = <<~HELP
+      NAME
+          new
+      
+      USAGE
+          bolt plan new <plan> [options]
+      
+      DESCRIPTION
+          Create a new plan in the current project.
+
+      EXAMPLES
+          bolt plan new myproject::myplan
+    HELP
+
     PLAN_RUN_HELP = <<~HELP
       NAME
           run
@@ -594,13 +635,13 @@ module Bolt
             bolt task show canary
     HELP
 
-    attr_reader :warnings
+    attr_reader :deprecations
 
     def initialize(options)
       super()
 
       @options = options
-      @warnings = []
+      @deprecations = []
 
       separator "\nINVENTORY OPTIONS"
       define('-t', '--targets TARGETS',
@@ -738,16 +779,29 @@ module Bolt
         @options[:'save-rerun'] = save
       end
 
+      separator "\nREMOTE ENVIRONMENT OPTIONS"
+      define('--env-var ENVIRONMENT_VARIABLES', 'Environment variables to set on the target') do |envvar|
+        unless envvar.include?('=')
+          raise Bolt::CLIError, "Environment variables must be specified using 'myenvvar=key' format"
+        end
+        @options[:env_vars] ||= {}
+        @options[:env_vars].store(*envvar.split('=', 2))
+      end
+
       separator "\nTRANSPORT OPTIONS"
       define('--transport TRANSPORT', TRANSPORTS.keys.map(&:to_s),
              "Specify a default transport: #{TRANSPORTS.keys.join(', ')}") do |t|
         @options[:transport] = t
       end
-      define('--ssh-command EXEC', "Executable to use instead of the net-ssh ruby library. ",
+      define('--[no-]native-ssh', 'Whether to shell out to native SSH or use the net-ssh Ruby library.',
+             'This option is experimental') do |bool|
+        @options[:'native-ssh'] = bool
+      end
+      define('--ssh-command EXEC', "Executable to use instead of the net-ssh Ruby library. ",
              "This option is experimental.") do |exec|
         @options[:'ssh-command'] = exec
       end
-      define('--copy-command EXEC', "Command to copy files to remote hosts if using external SSH. ",
+      define('--copy-command EXEC', "Command to copy files to remote hosts if using native SSH. ",
              "This option is experimental.") do |exec|
         @options[:'copy-command'] = exec
       end
@@ -805,7 +859,8 @@ module Bolt
         @options[:debug] = true
         # We don't actually set '--log-level debug' here, but once the options are evaluated by
         # the config class the end result is the same.
-        @warnings << { msg: "Command line option '--debug' is deprecated, set '--log-level debug' instead." }
+        msg = "Command line option '--debug' is deprecated, set '--log-level debug' instead."
+        @deprecations << { type: 'Using --debug instead of --log-level debug', msg: msg }
       end
       define('--log-level LEVEL',
              "Set the log level for the console. Available options are",