bolt 2.8.0 → 2.12.0

data/lib/bolt/result.rb

@@ -40,18 +40,23 @@ module Bolt
     end
 
     def self.for_task(target, stdout, stderr, exit_code, task)
-      begin
-        value = JSON.parse(stdout)
-        unless value.is_a? Hash
-          value = nil
-        end
-      rescue JSON::ParserError
-        value = nil
-      end
-      value ||= { '_output' => stdout }
+      stdout.force_encoding('utf-8') unless stdout.encoding == Encoding::UTF_8
+      value = if stdout.valid_encoding?
+                parse_hash(stdout) || { '_output' => stdout }
+              else
+                { '_error' => { 'kind' => 'puppetlabs.tasks/task-error',
+                                'issue_code' => 'TASK_ERROR',
+                                'msg' => 'The task result contained invalid UTF-8 on stdout',
+                                'details' => {} } }
+              end
+
       if exit_code != 0 && value['_error'].nil?
         msg = if stdout.empty?
-                "The task failed with exit code #{exit_code}:\n#{stderr}"
+                if stderr.empty?
+                  "The task failed with exit code #{exit_code} and no output"
+                else
+                  "The task failed with exit code #{exit_code} and no stdout, but stderr contained:\n#{stderr}"
+                end
               else
                 "The task failed with exit code #{exit_code}"
               end
@@ -63,6 +68,13 @@ module Bolt
       new(target, value: value, action: 'task', object: task)
     end
 
+    def self.parse_hash(string)
+      value = JSON.parse(string)
+      value if value.is_a? Hash
+    rescue JSON::ParserError
+      nil
+    end
+
     def self.for_upload(target, source, destination)
       new(target, message: "Uploaded '#{source}' to '#{target.host}:#{destination}'", action: 'upload', object: source)
     end
@@ -110,18 +122,8 @@ module Bolt
       message && !message.strip.empty?
     end
 
-    def status_hash
-      {
-        target: @target.name,
-        action: action,
-        object: object,
-        status: status,
-        value: @value
-      }
-    end
-
     def generic_value
-      value.reject { |k, _| %w[_error _output].include? k }
+      safe_value.reject { |k, _| %w[_error _output].include? k }
     end
 
     def eql?(other)
@@ -139,15 +141,36 @@ module Bolt
     end
 
     def to_json(opts = nil)
-      status_hash.to_json(opts)
+      to_data.to_json(opts)
     end
 
     def to_s
       to_json
     end
 
+    # This is the value with all non-UTF-8 characters removed, suitable for
+    # printing or converting to JSON. It *should* only be possible to have
+    # non-UTF-8 characters in stdout/stderr keys as they are not allowed from
+    # tasks but we scrub the whole thing just in case.
+    def safe_value
+      Bolt::Util.walk_vals(value) do |val|
+        if val.is_a?(String)
+          # Replace invalid bytes with hex codes, ie. \xDE\xAD\xBE\xEF
+          val.scrub { |c| c.bytes.map { |b| "\\x" + b.to_s(16).upcase }.join }
+        else
+          val
+        end
+      end
+    end
+
     def to_data
-      Bolt::Util.walk_keys(status_hash, &:to_s)
+      {
+        "target" => @target.name,
+        "action" => action,
+        "object" => object,
+        "status" => status,
+        "value" => safe_value
+      }
     end
 
     def status

data/lib/bolt/result_set.rb

@@ -99,17 +99,14 @@ module Bolt
       self.class == other.class && @results == other.results
     end
 
-    def to_a
-      @results.map(&:status_hash)
-    end
-
     def to_json(opts = nil)
-      @results.map(&:status_hash).to_json(opts)
+      to_data.to_json(opts)
     end
 
     def to_data
       @results.map(&:to_data)
     end
+    alias to_a to_data
 
     def to_s
       to_json

data/lib/bolt/shell/bash.rb

@@ -353,9 +353,9 @@ module Bolt
         # Chunks of this size will be read in one iteration
         index = 0
         timeout = 0.1
+        result_output = Bolt::Node::Output.new
 
         inp, out, err, t = conn.execute(command_str)
-        result_output = Bolt::Node::Output.new
         read_streams = { out => String.new,
                          err => String.new }
         write_stream = in_buffer.empty? ? [] : [inp]

data/lib/bolt/shell/powershell.rb

@@ -267,10 +267,18 @@ module Bolt
 
         result = Bolt::Node::Output.new
         inp.close
-        out.binmode
-        err.binmode
-        stdout = Thread.new { result.stdout << out.read }
-        stderr = Thread.new { result.stderr << err.read }
+        stdout = Thread.new do
+          # Set to binmode to preserve \r\n line endings, but save and restore
+          # the proper encoding so the string isn't later misinterpreted
+          encoding = out.external_encoding
+          out.binmode
+          result.stdout << out.read.force_encoding(encoding)
+        end
+        stderr = Thread.new do
+          encoding = err.external_encoding
+          err.binmode
+          result.stderr << err.read.force_encoding(encoding)
+        end
 
         stdout.join
         stderr.join

data/lib/bolt/target.rb

@@ -31,7 +31,8 @@ module Bolt
                                 facts = nil,
                                 vars = nil,
                                 features = nil,
-                                plugin_hooks = nil)
+                                plugin_hooks = nil,
+                                resources = nil)
       from_asserted_hash('uri' => uri)
     end
     # rubocop:enable Lint/UnusedMethodArgument
@@ -75,6 +76,16 @@ module Bolt
       inventory_target.target_alias
     end
 
+    def resources
+      inventory_target.resources
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+    def set_resource(resource)
+      inventory_target.set_resource(resource)
+    end
+    # rubocop:enable Naming/AccessorMethodName
+
     def to_h
       options.to_h.merge(
         'name' => name,

data/lib/bolt/transport/ssh.rb

@@ -16,13 +16,16 @@ module Bolt
         rescue LoadError
           logger.debug("Authentication method 'gssapi-with-mic' (Kerberos) is not available.")
         end
-
         @transport_logger = Logging.logger[Net::SSH]
         @transport_logger.level = :warn
       end
 
       def with_connection(target)
-        conn = Connection.new(target, @transport_logger)
+        conn = if target.transport_config['ssh-command']
+                 ExecConnection.new(target)
+               else
+                 Connection.new(target, @transport_logger)
+               end
         conn.connect
         yield conn
       ensure
@@ -37,3 +40,4 @@ module Bolt
 end
 
 require 'bolt/transport/ssh/connection'
+require 'bolt/transport/ssh/exec_connection'

data/lib/bolt/transport/ssh/connection.rb

@@ -207,6 +207,10 @@ module Bolt
             err_wr.close
           end
           [in_wr, out_rd, err_rd, th]
+        rescue Errno::EMFILE => e
+          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+            "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+          raise Bolt::Error.new(msg, 'bolt/too-many-files')
         end
 
         def copy_file(source, destination)

data/lib/bolt/transport/ssh/exec_connection.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Bolt
+  module Transport
+    class SSH < Simple
+      class ExecConnection
+        attr_reader :user, :target
+
+        def initialize(target)
+          raise Bolt::ValidationError, "Target #{target.safe_name} does not have a host" unless target.host
+
+          @target = target
+          ssh_config = Net::SSH::Config.for(target.host)
+          @user = @target.user || ssh_config[:user] || Etc.getlogin
+          @logger = Logging.logger[self]
+        end
+
+        # This is used to verify we can connect to targets with `connected?`
+        def connect
+          cmd = build_ssh_command('exit')
+          _, err, stat = Open3.capture3(*cmd)
+          unless stat.success?
+            raise Bolt::Node::ConnectError.new(
+              "Failed to connect to #{@target.safe_name}: #{err}",
+              'CONNECT_ERROR'
+            )
+          end
+        end
+
+        def disconnect; end
+
+        def shell
+          Bolt::Shell::Bash.new(@target, self)
+        end
+
+        def userhost
+          "#{@user}@#{@target.host}"
+        end
+
+        def ssh_opts
+          cmd = []
+          # BatchMode is SSH's noninteractive option: if key authentication
+          # fails it will error out instead of falling back to password prompt
+          cmd += %w[-o BatchMode=yes]
+          cmd += %W[-o Port=#{@target.port}] if @target.port
+
+          if @target.transport_config.key?('host-key-check')
+            hkc = @target.transport_config['host-key-check'] ? 'yes' : 'no'
+            cmd += %W[-o StrictHostKeyChecking=#{hkc}]
+          end
+
+          if (key = target.transport_config['private-key'])
+            cmd += ['-i', key]
+          end
+          cmd
+        end
+
+        def build_ssh_command(command)
+          ssh_conf = @target.transport_config['ssh-command']
+          ssh_cmd = Array(ssh_conf)
+          ssh_cmd += ssh_opts
+          ssh_cmd << userhost
+          ssh_cmd << command
+        end
+
+        def copy_file(source, dest)
+          @logger.debug { "Uploading #{source}, to #{userhost}:#{dest}" } unless source.is_a?(StringIO)
+
+          cp_conf = @target.transport_config['copy-command'] || ["scp", "-r"]
+          cp_cmd = Array(cp_conf)
+          cp_cmd += ssh_opts
+
+          _, err, stat = if source.is_a?(StringIO)
+                           Tempfile.create(File.basename(dest)) do |f|
+                             f.write(source.read)
+                             f.close
+                             cp_cmd << f.path
+                             cp_cmd << "#{userhost}:#{Shellwords.escape(dest)}"
+                             Open3.capture3(*cp_cmd)
+                           end
+                         else
+                           cp_cmd << source
+                           cp_cmd << "#{userhost}:#{Shellwords.escape(dest)}"
+                           Open3.capture3(*cp_cmd)
+                         end
+
+          if stat.success?
+            @logger.debug "Successfully uploaded #{source} to #{dest}"
+          else
+            message = "Could not copy file to #{dest}: #{err}"
+            raise Bolt::Node::FileError.new(message, 'COPY_ERROR')
+          end
+        end
+
+        def execute(command)
+          cmd_array = build_ssh_command(command)
+          Open3.popen3(*cmd_array)
+        end
+
+        # This is used by the Bash shell to decide whether to `cd` before
+        # executing commands as a run-as user
+        def reset_cwd?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/bolt/transport/winrm/connection.rb

@@ -108,8 +108,8 @@ module Bolt
           # it will fail if the shell attempts to provide stdin
           inp.close
 
-          out_rd, out_wr = IO.pipe
-          err_rd, err_wr = IO.pipe
+          out_rd, out_wr = IO.pipe('UTF-8')
+          err_rd, err_wr = IO.pipe('UTF-8')
           th = Thread.new do
             result = @session.run(command)
             out_wr << result.stdout
@@ -120,6 +120,10 @@ module Bolt
           end
 
           [inp, out_rd, err_rd, th]
+        rescue Errno::EMFILE => e
+          msg = "#{e.message}. This may be resolved by increasing your user limit "\
+            "with 'ulimit -n 1024'. See https://puppet.com/docs/bolt/latest/bolt_known_issues.html for details."
+          raise Bolt::Error.new(msg, 'bolt/too-many-files')
         rescue StandardError
           @logger.debug { "Command aborted" }
           raise

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '2.8.0'
+  VERSION = '2.12.0'
 end

data/lib/bolt_server/pe/pal.rb

@@ -51,50 +51,13 @@ module BoltServer
         basemodulepath = plan_executor_config['basemodulepath'] || "#{codedir}/modules:/opt/puppetlabs/puppet/modules"
 
         with_pe_pal_init_settings(codedir, environmentpath, basemodulepath) do
-          modulepath_dirs = []
-          modulepath_setting_from_bolt = nil
           environment = Puppet.lookup(:environments).get!(environment_name)
-          path_to_env = environment.configuration.path_to_env
-
-          # In the instance where the environment is "production" but no production dir
-          # exists, the lookup will succeed, but the configuration will be mostly empty.
-          # For other environments the lookup will fail, but for production we don't
-          # want cryptic messages sent to the user about combining `nil` with a string.
-          # Thus if we do get here and `path_to_env` is empty, just assume it's the
-          # default production environment and continue.
-          #
-          # This should hopefully match puppet's behavior for the default 'production'
-          # environment: _technically_ that environment always exists, but if the dir
-          # isn't there it won't find the module and fail with "plan not found" rather
-          # than "environment doesn't exist"
-          if path_to_env
-            bolt_yaml = File.join(environment.configuration.path_to_env, 'bolt.yaml')
-            modulepath_setting_from_bolt = Bolt::Util.read_optional_yaml_hash(bolt_yaml, 'config')['modulepath']
-          end
-
-          # If we loaded a bolt.yaml in the environment root and it contained a modulepath setting:
-          # we will use that modulepath rather than the one loaded through puppet. modulepath will
-          # be the _only_ setting that will work from bolt.yaml in plans in PE.
-          if modulepath_setting_from_bolt
-            modulepath_setting_from_bolt.split(File::PATH_SEPARATOR).each do |path|
-              if Pathname.new(path).absolute? && File.exist?(path)
-                modulepath_dirs << path
-              elsif File.exist?(File.join(path_to_env, path))
-                modulepath_dirs << File.join(path_to_env, path)
-              end
-            end
-
-            # Append the basemodulepath to include "built-in" modules.
-            modulepath_dirs.concat(basemodulepath.split(File::PATH_SEPARATOR))
-          else
-            modulepath_dirs = environment.modulepath
-          end
-
           # A new modulepath is created from scratch (rather than using super's @modulepath)
           # so that we can have full control over all the entries in modulepath. In the future
           # it's likely we will need to preceed _both_ Bolt::PAL::BOLTLIB_PATH _and_
           # Bolt::PAL::MODULES_PATH which would be more complex if we tried to use @modulepath since
           # we need to append our modulepaths and exclude modules shiped in bolt gem code
+          modulepath_dirs = environment.modulepath
           @original_modulepath = modulepath_dirs
           @modulepath = [PE_BOLTLIB_PATH, Bolt::PAL::BOLTLIB_PATH, *modulepath_dirs]
         end

data/lib/bolt_server/transport_app.rb

@@ -57,8 +57,8 @@ module BoltServer
     end
 
     def scrub_stack_trace(result)
-      if result.dig(:value, '_error', 'details', 'stack_trace')
-        result[:value]['_error']['details'].reject! { |k| k == 'stack_trace' }
+      if result.dig('value', '_error', 'details', 'stack_trace')
+        result['value']['_error']['details'].reject! { |k| k == 'stack_trace' }
       end
       result
     end
@@ -87,14 +87,14 @@ module BoltServer
     # If the `result_set` contains only one item, it will be returned
     # as a single result object. Set `aggregate` to treat it as a set
     # of results with length 1 instead.
-    def result_set_to_status_hash(result_set, aggregate: false)
+    def result_set_to_data(result_set, aggregate: false)
       scrubbed_results = result_set.map do |result|
-        scrub_stack_trace(result.status_hash)
+        scrub_stack_trace(result.to_data)
       end
 
       if aggregate || scrubbed_results.length > 1
         # For actions that act on multiple targets, construct a status hash for the aggregate result
-        all_succeeded = scrubbed_results.all? { |r| r[:status] == 'success' }
+        all_succeeded = scrubbed_results.all? { |r| r['status'] == 'success' }
         {
           status: all_succeeded ? 'success' : 'failure',
           result: scrubbed_results
@@ -297,7 +297,7 @@ module BoltServer
       return [400, error.to_json] unless error.nil?
 
       aggregate = body['target'].nil?
-      [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
+      [200, result_set_to_data(result_set, aggregate: aggregate).to_json]
     end
 
     def make_winrm_target(target_hash)
@@ -337,7 +337,7 @@ module BoltServer
       return [400, error.to_json] if error
 
       aggregate = body['target'].nil?
-      [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
+      [200, result_set_to_data(result_set, aggregate: aggregate).to_json]
     end
 
     # Fetches the metadata for a single plan