bolt 2.19.0 → 2.20.0

data/lib/bolt/config.rb

@@ -499,7 +499,7 @@ module Bolt
     end
 
     def matching_paths(paths)
-      [*paths].map { |p| Dir.glob([p, casefold(p)]) }.flatten.uniq.reject { |p| [*paths].include?(p) }
+      Array(paths).map { |p| Dir.glob([p, casefold(p)]) }.flatten.uniq.reject { |p| Array(paths).include?(p) }
     end
 
     private def casefold(path)

data/lib/bolt/config/options.rb

@@ -275,11 +275,25 @@ module Bolt
               type: String,
               _example: "/etc/puppetlabs/puppet/ssl/certs/my-host.example.com.pem"
             },
+            "connect_timeout" => {
+              description: "How long to wait in seconds when establishing connections with PuppetDB.",
+              type: Integer,
+              minimum: 1,
+              _default: 60,
+              _example: 120
+            },
             "key" => {
               description: "The private key for the certificate.",
               type: String,
               _example: "/etc/puppetlabs/puppet/ssl/private_keys/my-host.example.com.pem"
             },
+            "read_timeout" => {
+              description: "How long to wait in seconds for a response from PuppetDB.",
+              type: Integer,
+              minimum: 1,
+              _default: 60,
+              _example: 120
+            },
             "server_urls" => {
               description: "An array containing the PuppetDB host to connect to. Include the protocol `https` "\
                            "and the port, which is usually `8081`. For example, "\

data/lib/bolt/executor.rb

@@ -320,6 +320,21 @@ module Bolt
       end
     end
 
+    def download_file(targets, source, destination, options = {})
+      description = options.fetch(:description, "file download from #{source} to #{destination}")
+      FileUtils.mkdir_p(destination)
+
+      log_action(description, targets) do
+        options[:run_as] = run_as if run_as && !options.key?(:run_as)
+
+        batch_execute(targets) do |transport, batch|
+          with_node_logging("Downloading file #{source} to #{destination}", batch) do
+            transport.batch_download(batch, source, destination, options, &method(:publish_event))
+          end
+        end
+      end
+    end
+
     def run_plan(scope, plan, params)
       plan.call_by_name_with_scope(scope, params, true)
     end

data/lib/bolt/inventory/group.rb

@@ -50,10 +50,11 @@ module Bolt
           # or it could be a name/alias of a target defined in another group.
           # We can't tell the difference until all groups have been resolved,
           # so we store the string on its own here and process it later.
-          if target.is_a?(String)
+          case target
+          when String
             @string_targets << target
           # Handle plugins at this level so that lookups cannot trigger recursive lookups
-          elsif target.is_a?(Hash)
+          when Hash
             add_target_definition(target)
           else
             raise ValidationError.new("Target entry must be a String or Hash, not #{target.class}", @name)

data/lib/bolt/inventory/inventory.rb

@@ -109,11 +109,12 @@ module Bolt
       private :resolve_name
 
       def expand_targets(targets)
-        if targets.is_a? Bolt::Target
+        case targets
+        when Bolt::Target
           targets
-        elsif targets.is_a? Array
+        when Array
           targets.map { |tish| expand_targets(tish) }
-        elsif targets.is_a? String
+        when String
           # Expand a comma-separated list
           targets.split(/[[:space:],]+/).reject(&:empty?).map do |name|
             ts = resolve_name(name)

data/lib/bolt/outputter/rainbow.rb

@@ -39,9 +39,10 @@ module Bolt
             a = string.chars.map do |c|
               case @state
               when :normal
-                if c == "\e"
+                case c
+                when "\e"
                   @state = :ansi
-                elsif c == "\n"
+                when "\n"
                   @line_color += 1
                   @color = @line_color
                   c

data/lib/bolt/pal.rb

@@ -150,7 +150,12 @@ module Bolt
       r = Puppet::Pal.in_tmp_environment('bolt', modulepath: @modulepath, facts: {}) do |pal|
         # Only load the project if it a) exists, b) has a name it can be loaded with
         bolt_project = @project if @project&.name
+        # Puppet currently won't receive the project unless it is a named project. Since
+        # the download_file plan function needs access to the project path, add it to the
+        # context.
+        bolt_project_data = @project
         Puppet.override(bolt_project: bolt_project,
+                        bolt_project_data: bolt_project_data,
                         yaml_plan_instantiator: Bolt::PAL::YamlPlan::Loader) do
           pal.with_script_compiler do |compiler|
             alias_types(compiler)
@@ -279,9 +284,10 @@ module Bolt
 
     def parse_params(type, object_name, params)
       in_bolt_compiler do |compiler|
-        if type == 'task'
+        case type
+        when 'task'
           param_spec = compiler.task_signature(object_name)&.task_hash&.dig('parameters')
-        elsif type == 'plan'
+        when 'plan'
           plan = compiler.plan_signature(object_name)
           param_spec = plan.params_type.elements&.each_with_object({}) { |t, h| h[t.name] = t.value_type } if plan
         end

data/lib/bolt/pal/yaml_plan/evaluator.rb

@@ -73,7 +73,7 @@ module Bolt
         end
 
         def upload_step(scope, step)
-          source = step['source']
+          source = step['upload'] || step['source']
           destination = step['destination']
           targets = step['targets'] || step['target']
           description = step['description']
@@ -83,6 +83,17 @@ module Bolt
           scope.call_function('upload_file', args)
         end
 
+        def download_step(scope, step)
+          source = step['download']
+          destination = step['destination']
+          targets = step['targets'] || step['target']
+          description = step['description']
+
+          args = [source, destination, targets]
+          args << description if description
+          scope.call_function('download_file', args)
+        end
+
         def eval_step(_scope, step)
           step['eval']
         end
@@ -145,6 +156,12 @@ module Bolt
             Bolt::Logger.deprecation_warning("Using 'target' parameter for YAML plan steps, not 'targets'", msg)
           end
 
+          if plan.steps.any? { |step| step.body.key?('source') }
+            msg = "The 'source' parameter for YAML plan upload steps is deprecated and will be removed "\
+                  "in a future version of Bolt. Use the 'upload' parameter instead."
+            Bolt::Logger.deprecation_warning("Using 'source' parameter for YAML upload steps, not 'upload'", msg)
+          end
+
           plan_result = closure_scope.with_local_scope(args_hash) do |scope|
             plan.steps.each do |step|
               step_result = dispatch_step(scope, step)

data/lib/bolt/pal/yaml_plan/step.rb

@@ -12,7 +12,7 @@ module Bolt
           Set['name', 'description', 'target', 'targets']
         end
 
-        STEP_KEYS = %w[command script task plan source destination eval resources].freeze
+        STEP_KEYS = %w[command script task plan source destination eval resources upload download].freeze
 
         def self.create(step_body, step_number)
           type_keys = (STEP_KEYS & step_body.keys)
@@ -22,8 +22,10 @@ module Bolt
           when 1
             type = type_keys.first
           else
-            if type_keys.to_set == Set['source', 'destination']
+            if [Set['source', 'destination'], Set['upload', 'destination']].include?(type_keys.to_set)
               type = 'upload'
+            elsif type_keys.to_set == Set['download', 'destination']
+              type = 'download'
             else
               raise step_error("Multiple action keys detected: #{type_keys.inspect}", step_body['name'], step_number)
             end
@@ -89,6 +91,12 @@ module Bolt
             missing_keys -= ['targets']
           end
 
+          # Handle cases where upload step uses deprecated 'source' key instead of 'upload'
+          # TODO: Remove when 'source' is removed
+          if body.include?('source')
+            missing_keys -= ['upload']
+          end
+
           if missing_keys.any?
             error_message = "The #{step_type.inspect} step requires: #{missing_keys.to_a.inspect} key(s)"
             err = step_error(error_message, body['name'], step_number)
@@ -156,3 +164,4 @@ require 'bolt/pal/yaml_plan/step/resources'
 require 'bolt/pal/yaml_plan/step/script'
 require 'bolt/pal/yaml_plan/step/task'
 require 'bolt/pal/yaml_plan/step/upload'
+require 'bolt/pal/yaml_plan/step/download'

data/lib/bolt/pal/yaml_plan/step/download.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Bolt
+  class PAL
+    class YamlPlan
+      class Step
+        class Download < Step
+          def self.allowed_keys
+            super + Set['download', 'destination']
+          end
+
+          def self.required_keys
+            Set['download', 'destination', 'targets']
+          end
+
+          def initialize(step_body)
+            super
+            @source = step_body['download']
+            @destination = step_body['destination']
+          end
+
+          def transpile
+            code = String.new("  ")
+            code << "$#{@name} = " if @name
+
+            fn = 'download_file'
+            args = [@source, @destination, @targets]
+            args << @description if @description
+
+            code << function_call(fn, args)
+
+            code << "\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/pal/yaml_plan/step/upload.rb

@@ -6,16 +6,16 @@ module Bolt
       class Step
         class Upload < Step
           def self.allowed_keys
-            super + Set['source', 'destination']
+            super + Set['source', 'destination', 'upload']
           end
 
           def self.required_keys
-            Set['destination', 'source', 'targets']
+            Set['upload', 'destination', 'targets']
           end
 
           def initialize(step_body)
             super
-            @source = step_body['source']
+            @source = step_body['upload'] || step_body['source']
             @destination = step_body['destination']
           end
 

data/lib/bolt/plugin/puppetdb.rb

@@ -85,7 +85,8 @@ module Bolt
 
       def resolve_facts(config, certname, target_data)
         Bolt::Util.walk_vals(config) do |value|
-          if value.is_a?(String)
+          case value
+          when String
             if value == 'certname'
               certname
             else
@@ -94,7 +95,7 @@ module Bolt
               # If there's no fact data this will be nil
               data&.fetch('value', nil)
             end
-          elsif value.is_a?(Array) || value.is_a?(Hash)
+          when Array, Hash
             value
           else
             raise FactLookupError.new(value, "fact lookups must be a string")

data/lib/bolt/project.rb

@@ -17,7 +17,7 @@ module Bolt
 
     attr_reader :path, :data, :config_file, :inventory_file, :modulepath, :hiera_config,
                 :puppetfile, :rerunfile, :type, :resource_types, :warnings, :project_file,
-                :deprecations
+                :deprecations, :downloads
 
     def self.default_project
       create_project(File.expand_path(File.join('~', '.puppetlabs', 'bolt')), 'user')
@@ -81,6 +81,7 @@ module Bolt
       @rerunfile = @path + '.rerun.json'
       @resource_types = @path + '.resource_types'
       @type = type
+      @downloads = @path + 'downloads'
 
       tc = Bolt::Config::INVENTORY_OPTIONS.keys & raw_data.keys
       if tc.any?

data/lib/bolt/puppetdb/client.rb

@@ -96,6 +96,8 @@ module Bolt
         @http = HTTPClient.new
         @http.ssl_config.set_client_cert_file(@config.cert, @config.key) if @config.cert
         @http.ssl_config.add_trust_ca(@config.cacert)
+        @http.connect_timeout = @config.connect_timeout if @config.connect_timeout
+        @http.receive_timeout = @config.read_timeout if @config.read_timeout
 
         @http
       end

data/lib/bolt/puppetdb/config.rb

@@ -132,6 +132,22 @@ module Bolt
         end
       end
 
+      def connect_timeout
+        validate_timeout('connect_timeout')
+        @settings['connect_timeout']
+      end
+
+      def read_timeout
+        validate_timeout('read_timeout')
+        @settings['read_timeout']
+      end
+
+      def validate_timeout(timeout)
+        unless @settings[timeout].nil? || (@settings[timeout].is_a?(Integer) && @settings[timeout] > 0)
+          raise Bolt::PuppetDBError, "#{timeout} must be a positive integer, received #{@settings[timeout]}"
+        end
+      end
+
       def to_hash
         @settings.dup
       end

data/lib/bolt/result.rb

@@ -79,6 +79,13 @@ module Bolt
       new(target, message: "Uploaded '#{source}' to '#{target.host}:#{destination}'", action: 'upload', object: source)
     end
 
+    def self.for_download(target, source, destination, download)
+      msg   = "Downloaded '#{target.host}:#{source}' to '#{destination}'"
+      value = { 'path' => download }
+
+      new(target, value: value, message: msg, action: 'download', object: source)
+    end
+
     # Satisfies the Puppet datatypes API
     def self.from_asserted_args(target, value)
       new(target, value: value)

data/lib/bolt/shell/bash.rb

@@ -37,7 +37,7 @@ module Bolt
           with_tmpdir do |dir|
             basename = File.basename(source)
             tmpfile = File.join(dir.to_s, basename)
-            conn.copy_file(source, tmpfile)
+            conn.upload_file(source, tmpfile)
             # pass over file ownership if we're using run-as to be a different user
             dir.chown(run_as)
             result = execute(['mv', '-f', tmpfile, destination], sudoable: true)
@@ -50,6 +50,27 @@ module Bolt
         end
       end
 
+      def download(source, destination, options = {})
+        running_as(options[:run_as]) do
+          # Target OS may be either Unix or Windows. Without knowing the target OS before-hand
+          # we can't assume whether the path separator is '/' or '\'. Assume we're connecting
+          # to a target with Unix and then check if the path exists after downloading.
+          download = File.join(destination, Bolt::Util.unix_basename(source))
+
+          conn.download_file(source, destination, download)
+
+          # If the download path doesn't exist, then the file was likely downloaded from Windows
+          # using a source path with backslashes (e.g. 'C:\Users\Administrator\foo'). The file
+          # should be saved to the expected location, so update the download path assuming a
+          # Windows basename so the result shows the correct local path.
+          unless File.exist?(download)
+            download = File.join(destination, Bolt::Util.windows_basename(source))
+          end
+
+          Bolt::Result.for_download(target, source, destination, download)
+        end
+      end
+
       def run_script(script, arguments, options = {})
         # unpack any Sensitive data
         arguments = unwrap_sensitive_args(arguments)
@@ -95,7 +116,7 @@ module Bolt
               task_dir = File.join(dir.to_s, task.tasks_dir)
               dir.mkdirs([task.tasks_dir] + extra_files.map { |file| File.dirname(file['name']) })
               extra_files.each do |file|
-                conn.copy_file(file['path'], File.join(dir.to_s, file['name']))
+                conn.upload_file(file['path'], File.join(dir.to_s, file['name']))
               end
             end
 
@@ -257,7 +278,7 @@ module Bolt
       def write_executable(dir, file, filename = nil)
         filename ||= File.basename(file)
         remote_path = File.join(dir.to_s, filename)
-        conn.copy_file(file, remote_path)
+        conn.upload_file(file, remote_path)
         make_executable(remote_path)
         remote_path
       end
@@ -314,7 +335,6 @@ module Bolt
           sudo_str = if use_sudo
                        sudo_exec = target.options['sudo-executable'] || "sudo"
                        sudo_flags = [sudo_exec, "-S", "-H", "-u", run_as, "-p", sudo_prompt]
-                       sudo_flags += ["-E"] if options[:environment]
                        Shellwords.shelljoin(sudo_flags)
                      else
                        Shellwords.shelljoin(@target.options['run-as-command'] + [run_as])

data/lib/bolt/shell/powershell.rb

@@ -85,7 +85,7 @@ module Bolt
         filename ||= File.basename(file)
         validate_extensions(File.extname(filename))
         destination = "#{dir}\\#{filename}"
-        conn.copy_file(file, destination)
+        conn.upload_file(file, destination)
         destination
       end
 
@@ -164,10 +164,16 @@ module Bolt
       end
 
       def upload(source, destination, _options = {})
-        conn.copy_file(source, destination)
+        conn.upload_file(source, destination)
         Bolt::Result.for_upload(target, source, destination)
       end
 
+      def download(source, destination, _options = {})
+        download = File.join(destination, Bolt::Util.windows_basename(source))
+        conn.download_file(source, destination, download)
+        Bolt::Result.for_download(target, source, destination, download)
+      end
+
       def run_command(command, options = {})
         command = [*env_declarations(options[:env_vars]), command].join("\r\n") if options[:env_vars]
 
@@ -220,7 +226,7 @@ module Bolt
             task_dir = File.join(dir, task.tasks_dir)
             mkdirs([task_dir] + extra_files.map { |file| File.join(dir, File.dirname(file['name'])) })
             extra_files.each do |file|
-              conn.copy_file(file['path'], File.join(dir, file['name']))
+              conn.upload_file(file['path'], File.join(dir, file['name']))
             end
           end
 
@@ -262,7 +268,7 @@ module Bolt
           return with_tmpdir do |dir|
             command += "\r\nif (!$?) { if($LASTEXITCODE) { exit $LASTEXITCODE } else { exit 1 } }"
             script_file = File.join(dir, "#{SecureRandom.uuid}_wrapper.ps1")
-            conn.copy_file(StringIO.new(command), script_file)
+            conn.upload_file(StringIO.new(command), script_file)
             args = escape_arguments([script_file])
             script_invocation = ['powershell.exe', *PS_ARGS, '-File', *args].join(' ')
             execute(script_invocation)