bolt 2.7.0 → 2.11.1

data/lib/bolt/result.rb

@@ -40,18 +40,23 @@ module Bolt
     end
 
     def self.for_task(target, stdout, stderr, exit_code, task)
-      begin
-        value = JSON.parse(stdout)
-        unless value.is_a? Hash
-          value = nil
-        end
-      rescue JSON::ParserError
-        value = nil
-      end
-      value ||= { '_output' => stdout }
+      stdout.force_encoding('utf-8') unless stdout.encoding == Encoding::UTF_8
+      value = if stdout.valid_encoding?
+                parse_hash(stdout) || { '_output' => stdout }
+              else
+                { '_error' => { 'kind' => 'puppetlabs.tasks/task-error',
+                                'issue_code' => 'TASK_ERROR',
+                                'msg' => 'The task result contained invalid UTF-8 on stdout',
+                                'details' => {} } }
+              end
+
       if exit_code != 0 && value['_error'].nil?
         msg = if stdout.empty?
-                "The task failed with exit code #{exit_code}:\n#{stderr}"
+                if stderr.empty?
+                  "The task failed with exit code #{exit_code} and no output"
+                else
+                  "The task failed with exit code #{exit_code} and no stdout, but stderr contained:\n#{stderr}"
+                end
               else
                 "The task failed with exit code #{exit_code}"
               end
@@ -63,6 +68,13 @@ module Bolt
       new(target, value: value, action: 'task', object: task)
     end
 
+    def self.parse_hash(string)
+      value = JSON.parse(string)
+      value if value.is_a? Hash
+    rescue JSON::ParserError
+      nil
+    end
+
     def self.for_upload(target, source, destination)
       new(target, message: "Uploaded '#{source}' to '#{target.host}:#{destination}'", action: 'upload', object: source)
     end
@@ -110,18 +122,8 @@ module Bolt
       message && !message.strip.empty?
     end
 
-    def status_hash
-      {
-        target: @target.name,
-        action: action,
-        object: object,
-        status: status,
-        value: @value
-      }
-    end
-
     def generic_value
-      value.reject { |k, _| %w[_error _output].include? k }
+      safe_value.reject { |k, _| %w[_error _output].include? k }
     end
 
     def eql?(other)
@@ -139,15 +141,36 @@ module Bolt
     end
 
     def to_json(opts = nil)
-      status_hash.to_json(opts)
+      to_data.to_json(opts)
     end
 
     def to_s
       to_json
     end
 
+    # This is the value with all non-UTF-8 characters removed, suitable for
+    # printing or converting to JSON. It *should* only be possible to have
+    # non-UTF-8 characters in stdout/stderr keys as they are not allowed from
+    # tasks but we scrub the whole thing just in case.
+    def safe_value
+      Bolt::Util.walk_vals(value) do |val|
+        if val.is_a?(String)
+          # Replace invalid bytes with hex codes, ie. \xDE\xAD\xBE\xEF
+          val.scrub { |c| c.bytes.map { |b| "\\x" + b.to_s(16).upcase }.join }
+        else
+          val
+        end
+      end
+    end
+
     def to_data
-      Bolt::Util.walk_keys(status_hash, &:to_s)
+      {
+        "target" => @target.name,
+        "action" => action,
+        "object" => object,
+        "status" => status,
+        "value" => safe_value
+      }
     end
 
     def status

data/lib/bolt/result_set.rb

@@ -99,17 +99,14 @@ module Bolt
       self.class == other.class && @results == other.results
     end
 
-    def to_a
-      @results.map(&:status_hash)
-    end
-
     def to_json(opts = nil)
-      @results.map(&:status_hash).to_json(opts)
+      to_data.to_json(opts)
     end
 
     def to_data
       @results.map(&:to_data)
     end
+    alias to_a to_data
 
     def to_s
       to_json

data/lib/bolt/secret.rb

@@ -1,17 +1,33 @@
 # frozen_string_literal: true
 
+require 'bolt/plugin'
+
 module Bolt
   class Secret
+    KNOWN_KEYS = {
+      'createkeys' => %w[keysize private_key public_key],
+      'encrypt'    => %w[public_key],
+      'decrypt'    => %w[private_key public_key]
+    }.freeze
+
     def self.execute(plugins, outputter, options)
-      plugin = options[:plugin] || 'pkcs7'
+      name   = options[:plugin] || 'pkcs7'
+      plugin = plugins.by_name(name)
+
+      unless plugin
+        raise Bolt::Plugin::PluginError::Unknown, name
+      end
+
       case options[:action]
       when 'createkeys'
-        plugins.get_hook(plugin, :secret_createkeys).call
+        opts = { 'force' => options[:force] }.compact
+        result = plugins.get_hook(name, :secret_createkeys).call(opts)
+        outputter.print_message(result)
       when 'encrypt'
-        encrypted = plugins.get_hook(plugin, :secret_encrypt).call('plaintext_value' => options[:object])
+        encrypted = plugins.get_hook(name, :secret_encrypt).call('plaintext_value' => options[:object])
         outputter.print_message(encrypted)
       when 'decrypt'
-        decrypted = plugins.get_hook(plugin, :secret_decrypt).call('encrypted_value' => options[:object])
+        decrypted = plugins.get_hook(name, :secret_decrypt).call('encrypted_value' => options[:object])
         outputter.print_message(decrypted)
       end
 

data/lib/bolt/shell/bash.rb

@@ -150,8 +150,14 @@ module Bolt
           end
         elsif err =~ /^#{@sudo_id}/
           if sudo_stdin
-            stdin.write("#{sudo_stdin}\n")
-            stdin.close
+            begin
+              stdin.write("#{sudo_stdin}\n")
+              stdin.close
+            # If a task has stdin as an input_method but doesn't actually read
+            # from stdin, the task may return and close the input stream before
+            # we finish writing
+            rescue Errno::EPIPE
+            end
           end
           ''
         else
@@ -347,9 +353,9 @@ module Bolt
         # Chunks of this size will be read in one iteration
         index = 0
         timeout = 0.1
+        result_output = Bolt::Node::Output.new
 
         inp, out, err, t = conn.execute(command_str)
-        result_output = Bolt::Node::Output.new
         read_streams = { out => String.new,
                          err => String.new }
         write_stream = in_buffer.empty? ? [] : [inp]
@@ -399,8 +405,9 @@ module Bolt
                 write_stream = []
               end
             end
-          # If a task has stdin as an input_method but doesn't actually
-          # read from stdin, the task may return and close the input stream
+          # If a task has stdin as an input_method but doesn't actually read
+          # from stdin, the task may return and close the input stream before
+          # we finish writing
           rescue Errno::EPIPE
             write_stream = []
           end

data/lib/bolt/shell/powershell.rb

@@ -267,10 +267,18 @@ module Bolt
 
         result = Bolt::Node::Output.new
         inp.close
-        out.binmode
-        err.binmode
-        stdout = Thread.new { result.stdout << out.read }
-        stderr = Thread.new { result.stderr << err.read }
+        stdout = Thread.new do
+          # Set to binmode to preserve \r\n line endings, but save and restore
+          # the proper encoding so the string isn't later misinterpreted
+          encoding = out.external_encoding
+          out.binmode
+          result.stdout << out.read.force_encoding(encoding)
+        end
+        stderr = Thread.new do
+          encoding = err.external_encoding
+          err.binmode
+          result.stderr << err.read.force_encoding(encoding)
+        end
 
         stdout.join
         stderr.join

data/lib/bolt/target.rb

@@ -31,7 +31,8 @@ module Bolt
                                 facts = nil,
                                 vars = nil,
                                 features = nil,
-                                plugin_hooks = nil)
+                                plugin_hooks = nil,
+                                resources = nil)
       from_asserted_hash('uri' => uri)
     end
     # rubocop:enable Lint/UnusedMethodArgument
@@ -75,6 +76,16 @@ module Bolt
       inventory_target.target_alias
     end
 
+    def resources
+      inventory_target.resources
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+    def set_resource(resource)
+      inventory_target.set_resource(resource)
+    end
+    # rubocop:enable Naming/AccessorMethodName
+
     def to_h
       options.to_h.merge(
         'name' => name,
@@ -155,5 +166,9 @@ module Bolt
       self.class.equal?(other.class) && @name == other.name
     end
     alias == eql?
+
+    def hash
+      @name.hash
+    end
   end
 end

data/lib/bolt/transport/base.rb

@@ -32,7 +32,7 @@ module Bolt
     # Transports that need their own batching, like the Orch transport, can
     # instead override the batches() method to split Targets into sets that can
     # be executed together, and override the batch_task() and related methods
-    # to execute a batch of nodes. In that case, those Transports should accept
+    # to execute a batch of targets. In that case, those Transports should accept
     # a block argument and call it with a :node_start event for each Target
     # before executing, and a :node_result event for each Target after
     # execution.
@@ -90,12 +90,12 @@ module Bolt
       # case and raises an error if it's not.
       def assert_batch_size_one(method, targets)
         if targets.length > 1
-          message = "#{self.class.name} must implement #{method} to support batches (got #{targets.length} nodes)"
+          message = "#{self.class.name} must implement #{method} to support batches (got #{targets.length} targets)"
           raise NotImplementedError, message
         end
       end
 
-      # Runs the given task on a batch of nodes.
+      # Runs the given task on a batch of targets.
       #
       # The default implementation only supports batches of size 1 and will fail otherwise.
       #
@@ -104,12 +104,28 @@ module Bolt
         assert_batch_size_one("batch_task()", targets)
         target = targets.first
         with_events(target, callback, 'task') do
-          @logger.debug { "Running task run '#{task}' on #{target.safe_name}" }
+          @logger.debug { "Running task '#{task.name}' on #{target.safe_name}" }
           run_task(target, task, arguments, options)
         end
       end
 
-      # Runs the given command on a batch of nodes.
+      # Runs the given task on a batch of targets with variable parameters.
+      #
+      # The default implementation only supports batches of size 1 and will fail otherwise.
+      #
+      # Transports may override this method to implment their own batch processing.
+      def batch_task_with(targets, task, target_mapping, options = {}, &callback)
+        assert_batch_size_one("batch_task_with()", targets)
+        target = targets.first
+        arguments = target_mapping[target]
+
+        with_events(target, callback, 'task') do
+          @logger.debug { "Running task '#{task.name}' on #{target.safe_name} with '#{arguments.to_json}'" }
+          run_task(target, task, arguments, options)
+        end
+      end
+
+      # Runs the given command on a batch of targets.
       #
       # The default implementation only supports batches of size 1 and will fail otherwise.
       #
@@ -123,7 +139,7 @@ module Bolt
         end
       end
 
-      # Runs the given script on a batch of nodes.
+      # Runs the given script on a batch of targets.
       #
       # The default implementation only supports batches of size 1 and will fail otherwise.
       #
@@ -137,7 +153,7 @@ module Bolt
         end
       end
 
-      # Uploads the given source file to the destination location on a batch of nodes.
+      # Uploads the given source file to the destination location on a batch of targets.
       #
       # The default implementation only supports batches of size 1 and will fail otherwise.
       #
@@ -157,7 +173,7 @@ module Bolt
       end
 
       # Split the given list of targets into a list of batches. The default
-      # implementation returns single-node batches.
+      # implementation returns single-target batches.
       #
       # Transports may override this method, and the corresponding batch_*
       # methods, to implement their own batch processing.

data/lib/bolt/transport/orch.rb

@@ -210,6 +210,10 @@ module Bolt
         end
       end
 
+      def batch_task_with(_targets, _task, _target_mapping, _options = {})
+        raise NotImplementedError, "pcp transport does not support run_task_with()"
+      end
+
       def batch_connected?(targets)
         resp = get_connection(targets.first.options).query_inventory(targets)
         resp['items'].all? { |node| node['connected'] }

data/lib/bolt/transport/ssh.rb

@@ -16,13 +16,16 @@ module Bolt
         rescue LoadError
           logger.debug("Authentication method 'gssapi-with-mic' (Kerberos) is not available.")
         end
-
         @transport_logger = Logging.logger[Net::SSH]
         @transport_logger.level = :warn
       end
 
       def with_connection(target)
-        conn = Connection.new(target, @transport_logger)
+        conn = if target.transport_config['ssh-command']
+                 ExecConnection.new(target)
+               else
+                 Connection.new(target, @transport_logger)
+               end
         conn.connect
         yield conn
       ensure
@@ -37,3 +40,4 @@ module Bolt
 end
 
 require 'bolt/transport/ssh/connection'
+require 'bolt/transport/ssh/exec_connection'

data/lib/bolt/transport/ssh/connection.rb

@@ -207,6 +207,10 @@ module Bolt
             err_wr.close
           end
           [in_wr, out_rd, err_rd, th]
+        rescue Errno::EMFILE => e
+          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+            "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+          raise Bolt::Error.new(msg, 'bolt/too-many-files')
         end
 
         def copy_file(source, destination)

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Bolt
+  module Transport
+    class SSH < Simple
+      class ExecConnection
+        attr_reader :user, :target
+
+        def initialize(target)
+          raise Bolt::ValidationError, "Target #{target.safe_name} does not have a host" unless target.host
+
+          @target = target
+          ssh_config = Net::SSH::Config.for(target.host)
+          @user = @target.user || ssh_config[:user] || Etc.getlogin
+          @logger = Logging.logger[self]
+        end
+
+        # This is used to verify we can connect to targets with `connected?`
+        def connect
+          cmd = build_ssh_command('exit')
+          _, err, stat = Open3.capture3(*cmd)
+          unless stat.success?
+            raise Bolt::Node::ConnectError.new(
+              "Failed to connect to #{@target.safe_name}: #{err}",
+              'CONNECT_ERROR'
+            )
+          end
+        end
+
+        def disconnect; end
+
+        def shell
+          Bolt::Shell::Bash.new(@target, self)
+        end
+
+        def userhost
+          "#{@user}@#{@target.host}"
+        end
+
+        def ssh_opts
+          cmd = []
+          # BatchMode is SSH's noninteractive option: if key authentication
+          # fails it will error out instead of falling back to password prompt
+          cmd += %w[-o BatchMode=yes]
+          cmd += %W[-o Port=#{@target.port}] if @target.port
+
+          if @target.transport_config.key?('host-key-check')
+            hkc = @target.transport_config['host-key-check'] ? 'yes' : 'no'
+            cmd += %W[-o StrictHostKeyChecking=#{hkc}]
+          end
+
+          if (key = target.transport_config['private-key'])
+            cmd += ['-i', key]
+          end
+          cmd
+        end
+
+        def build_ssh_command(command)
+          ssh_conf = @target.transport_config['ssh-command']
+          ssh_cmd = Array(ssh_conf)
+          ssh_cmd += ssh_opts
+          ssh_cmd << userhost
+          ssh_cmd << command
+        end
+
+        def copy_file(source, dest)
+          @logger.debug { "Uploading #{source}, to #{userhost}:#{dest}" } unless source.is_a?(StringIO)
+
+          cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
+          cp_cmd = Array(cp_conf)
+          cp_cmd += ssh_opts
+
+          _, err, stat = if source.is_a?(StringIO)
+                           Tempfile.create(File.basename(dest)) do |f|
+                             f.write(source.read)
+                             f.close
+                             cp_cmd << f.path
+                             cp_cmd << "#{userhost}:#{Shellwords.escape(dest)}"
+                             Open3.capture3(*cp_cmd)
+                           end
+                         else
+                           cp_cmd << source
+                           cp_cmd << "#{userhost}:#{Shellwords.escape(dest)}"
+                           Open3.capture3(*cp_cmd)
+                         end
+
+          if stat.success?
+            @logger.debug "Successfully uploaded #{source} to #{dest}"
+          else
+            message = "Could not copy file to #{dest}: #{err}"
+            raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
+          end
+        end
+
+        def execute(command)
+          cmd_array = build_ssh_command(command)
+          Open3.popen3(*cmd_array)
+        end
+
+        # This is used by the Bash shell to decide whether to `cd` before
+        # executing commands as a run-as user
+        def reset_cwd?
+          true
+        end
+      end
+    end
+  end
+end