bolt 2.44.0 → 3.4.0

data/lib/bolt/shell.rb

@@ -8,6 +8,22 @@ module Bolt
       @target = target
       @conn = conn
       @logger = Bolt::Logger.logger(@target.safe_name)
+
+      if Bolt::Logger.stream
+        Bolt::Logger.warn_once("stream_experimental",
+                               "The 'stream' option is experimental, and might "\
+                               "include breaking changes between minor versions.")
+        @stream_logger = Bolt::Logger.logger(:stream)
+        # Don't send stream messages to the parent logger
+        @stream_logger.additive = false
+
+        # Log stream messages without any other data or color
+        pattern = Logging.layouts.pattern(pattern: '%m\n')
+        @stream_logger.appenders = Logging.appenders.stdout(
+          'console',
+          layout: pattern
+        )
+      end
     end
 
     def run_command(*_args)

data/lib/bolt/shell/bash.rb

@@ -12,7 +12,6 @@ module Bolt
         super
 
         @run_as = nil
-
         @sudo_id = SecureRandom.uuid
         @sudo_password = @target.options['sudo-password'] || @target.password
       end
@@ -25,9 +24,7 @@ module Bolt
         running_as(options[:run_as]) do
           output = execute(command, environment: options[:env_vars], sudoable: true)
           Bolt::Result.for_command(target,
-                                   output.stdout.string,
-                                   output.stderr.string,
-                                   output.exit_code,
+                                   output.to_h,
                                    'command',
                                    command,
                                    position)
@@ -54,19 +51,35 @@ module Bolt
 
       def download(source, destination, options = {})
         running_as(options[:run_as]) do
-          # Target OS may be either Unix or Windows. Without knowing the target OS before-hand
-          # we can't assume whether the path separator is '/' or '\'. Assume we're connecting
-          # to a target with Unix and then check if the path exists after downloading.
           download = File.join(destination, Bolt::Util.unix_basename(source))
 
-          conn.download_file(source, destination, download)
+          # If using run-as, the file is copied to a tmpdir and chowned to the
+          # connecting user. This is a workaround for limitations in net-ssh that
+          # only allow for downloading files as the connecting user, which is a
+          # problem for users who cannot connect to targets as the root user.
+          # This temporary copy should *always* be deleted.
+          if run_as
+            with_tmpdir(force_cleanup: true) do |dir|
+              tmpfile = File.join(dir.to_s, Bolt::Util.unix_basename(source))
+
+              result = execute(['cp', '-r', source, dir.to_s], sudoable: true)
+
+              if result.exit_code != 0
+                message = "Could not copy file '#{source}' to temporary directory '#{dir}': #{result.stderr.string}"
+                raise Bolt::Node::FileError.new(message, 'CP_ERROR')
+              end
 
-          # If the download path doesn't exist, then the file was likely downloaded from Windows
-          # using a source path with backslashes (e.g. 'C:\Users\Administrator\foo'). The file
-          # should be saved to the expected location, so update the download path assuming a
-          # Windows basename so the result shows the correct local path.
-          unless File.exist?(download)
-            download = File.join(destination, Bolt::Util.windows_basename(source))
+              # We need to force the chown, otherwise this will just return
+              # without doing anything since the chown user is the same as the
+              # connecting user.
+              dir.chown(conn.user, force: true)
+
+              conn.download_file(tmpfile, destination, download)
+            end
+          # If not using run-as, we can skip creating a temporary copy and just
+          # download the file directly.
+          else
+            conn.download_file(source, destination, download)
           end
 
           Bolt::Result.for_download(target, source, destination, download)
@@ -83,9 +96,7 @@ module Bolt
             dir.chown(run_as)
             output = execute([path, *arguments], environment: options[:env_vars], sudoable: true)
             Bolt::Result.for_command(target,
-                                     output.stdout.string,
-                                     output.stderr.string,
-                                     output.exit_code,
+                                     output.to_h,
                                      'script',
                                      script,
                                      position)
@@ -134,18 +145,21 @@ module Bolt
 
             remote_task_path = write_executable(task_dir, executable)
 
+            execute_options[:stdin] = stdin
+
             # Avoid the horrors of passing data on stdin via a tty on multiple platforms
             # by writing a wrapper script that directs stdin to the task.
             if stdin && target.options['tty']
               wrapper = make_wrapper_stringio(remote_task_path, stdin, execute_options[:interpreter])
+              # Wrapper script handles interpreter and stdin. Delete these execute options
               execute_options.delete(:interpreter)
+              execute_options.delete(:stdin)
               execute_options[:wrapper] = true
               remote_task_path = write_executable(dir, wrapper, 'wrapper.sh')
             end
 
             dir.chown(run_as)
 
-            execute_options[:stdin] = stdin
             execute_options[:sudoable] = true if run_as
             output = execute(remote_task_path, **execute_options)
           end
@@ -291,12 +305,12 @@ module Bolt
 
       # A helper to create and delete a tmpdir on the remote system. Yields the
       # directory name.
-      def with_tmpdir
+      def with_tmpdir(force_cleanup: false)
         dir = make_tmpdir
         yield dir
       ensure
         if dir
-          if target.options['cleanup']
+          if target.options['cleanup'] || force_cleanup
             dir.delete
           else
             Bolt::Logger.warn("skip_cleanup", "Skipping cleanup of tmpdir #{dir}")
@@ -331,10 +345,15 @@ module Bolt
         # together multiple commands into a single sh invocation
         commands = [inject_interpreter(options[:interpreter], command)]
 
+        # Let the transport handle adding environment variables if it's custom.
         if options[:environment]
-          env_decl = options[:environment].map do |env, val|
-            "#{env}=#{Shellwords.shellescape(val)}"
-          end.join(' ')
+          if defined? conn.add_env_vars
+            conn.add_env_vars(options[:environment])
+          else
+            env_decl = options[:environment].map do |env, val|
+              "#{env}=#{Shellwords.shellescape(val)}"
+            end.join(' ')
+          end
         end
 
         if escalate
@@ -385,14 +404,24 @@ module Bolt
           # See if we can read from out or err, or write to in
           ready_read, ready_write, = select(read_streams.keys, write_stream, nil, timeout)
 
-          # Read from out and err
           ready_read&.each do |stream|
+            stream_name = stream == out ? 'out' : 'err'
             # Check for sudo prompt
-            read_streams[stream] << if use_sudo
-                                      check_sudo(stream, inp, options[:stdin])
-                                    else
-                                      stream.readpartial(CHUNK_SIZE)
-                                    end
+            to_print = if use_sudo
+                         check_sudo(stream, inp, options[:stdin])
+                       else
+                         stream.readpartial(CHUNK_SIZE)
+                       end
+
+            if !to_print.chomp.empty? && @stream_logger
+              formatted = to_print.lines.map do |msg|
+                "[#{@target.safe_name}] #{stream_name}: #{msg.chomp}"
+              end.join("\n")
+              @stream_logger.warn(formatted)
+            end
+
+            read_streams[stream]        << to_print
+            result_output.merged_output << to_print
           rescue EOFError
           end
 
@@ -440,9 +469,10 @@ module Bolt
         when 0
           @logger.trace { "Command `#{command_str}` returned successfully" }
         when 126
-          msg = "\n\nThis may be caused by the default tmpdir being mounted "\
+          msg = "\n\nThis might be caused by the default tmpdir being mounted "\
             "using 'noexec'. See http://pup.pt/task-failure for details and workarounds."
-          result_output.stderr << msg
+          result_output.stderr        << msg
+          result_output.merged_output << msg
           @logger.trace { "Command #{command_str} failed with exit code #{result_output.exit_code}" }
         else
           @logger.trace { "Command #{command_str} failed with exit code #{result_output.exit_code}" }

data/lib/bolt/shell/bash/tmpdir.rb

@@ -24,8 +24,8 @@ module Bolt
           end
         end
 
-        def chown(owner)
-          return if owner.nil? || owner == @owner
+        def chown(owner, force: false)
+          return if owner.nil? || (owner == @owner && !force)
 
           result = @shell.execute(['id', '-g', owner])
           if result.exit_code != 0

data/lib/bolt/shell/powershell.rb

@@ -23,8 +23,7 @@ module Bolt
           # This lets us know how many targets have Powershell 2, and lets the
           # user know how many targets they have with PS2
           msg = "Detected PowerShell 2 on one or more targets.\nPowerShell 2 "\
-            "is deprecated, and support will be removed in Bolt 3.0. See "\
-            "bolt-debug.log or run with '--log-level debug' to see the full "\
+            "is unsupported. See bolt-debug.log or run with '--log-level debug' to see the full "\
             "list of targets with PowerShell 2."
 
           Bolt::Logger.deprecate_once("powershell_2", msg)
@@ -196,9 +195,7 @@ module Bolt
         wrap_command = conn.is_a?(Bolt::Transport::Local::Connection)
         output = execute(command, wrap_command)
         Bolt::Result.for_command(target,
-                                 output.stdout.string,
-                                 output.stderr.string,
-                                 output.exit_code,
+                                 output.to_h,
                                  'command',
                                  command,
                                  position)
@@ -209,20 +206,23 @@ module Bolt
         arguments = unwrap_sensitive_args(arguments)
         with_tmpdir do |dir|
           script_path = write_executable(dir, script)
-          command = if powershell_file?(script_path)
+          command = if powershell_file?(script_path) && options[:pwsh_params]
+                      # Scripts run with pwsh_params can be run like tasks
+                      Snippets.ps_task(script_path, options[:pwsh_params])
+                    elsif powershell_file?(script_path)
                       Snippets.run_script(arguments, script_path)
                     else
                       path, args = *process_from_extension(script_path)
                       args += escape_arguments(arguments)
                       execute_process(path, args)
                     end
-          command = [*env_declarations(options[:env_vars]), command].join("\r\n") if options[:env_vars]
+          env_assignments = options[:env_vars] ? env_declarations(options[:env_vars]) : []
+          shell_init = options[:pwsh_params] ? Snippets.shell_init : ''
+
+          output = execute([shell_init, *env_assignments, command].join("\r\n"))
 
-          output = execute(command)
           Bolt::Result.for_command(target,
-                                   output.stdout.string,
-                                   output.stderr.string,
-                                   output.exit_code,
+                                   output.to_h,
                                    'script',
                                    script,
                                    position)
@@ -275,7 +275,12 @@ module Bolt
                               []
                             end
 
-          output = execute([Snippets.shell_init, *env_assignments, command].join("\n"))
+          output = execute([
+            Snippets.shell_init,
+            Snippets.append_ps_module_path(dir),
+            *env_assignments,
+            command
+          ].join("\n"))
 
           Bolt::Result.for_task(target, output.stdout.string,
                                 output.stderr.string,
@@ -306,12 +311,28 @@ module Bolt
           # the proper encoding so the string isn't later misinterpreted
           encoding = out.external_encoding
           out.binmode
-          result.stdout << out.read.force_encoding(encoding)
+          to_print = out.read.force_encoding(encoding)
+          if !to_print.chomp.empty? && @stream_logger
+            formatted = to_print.lines.map do |msg|
+              "[#{@target.safe_name}] out: #{msg.chomp}"
+            end.join("\n")
+            @stream_logger.warn(formatted)
+          end
+          result.stdout        << to_print
+          result.merged_output << to_print
         end
         stderr = Thread.new do
           encoding = err.external_encoding
           err.binmode
-          result.stderr << err.read.force_encoding(encoding)
+          to_print = err.read.force_encoding(encoding)
+          if !to_print.chomp.empty? && @stream_logger
+            formatted = to_print.lines.map do |msg|
+              "[#{@target.safe_name}] err: #{msg.chomp}"
+            end.join("\n")
+            @stream_logger.warn(formatted)
+          end
+          result.stderr        << to_print
+          result.merged_output << to_print
         end
 
         stdout.join

data/lib/bolt/shell/powershell/snippets.rb

@@ -55,27 +55,59 @@ module Bolt
             }
             #{build_arg_list}
 
+            switch -regex ( Get-ExecutionPolicy )
+            {
+              '^AllSigned'
+              {
+                if ((Get-AuthenticodeSignature -File "#{script_path}").Status -ne 'Valid') {
+                  $Host.UI.WriteErrorLine("Error: Target host Powershell ExecutionPolicy is set to ${_} and script '#{script_path}' does not contain a valid signature.")
+                  exit 1;
+                }
+              }
+              '^Restricted'
+              {
+                $Host.UI.WriteErrorLine("Error: Target host Powershell ExecutionPolicy is set to ${_} which denies running any scripts on the target.")
+                exit 1;
+              }
+            }
+
+            if([string]::IsNullOrEmpty($invokeArgs.ScriptBlock)){
+              $Host.UI.WriteErrorLine("Error: Failed to obtain scriptblock from '#{script_path}'. Running scripts might be disabled on this system. For more information, see about_Execution_Policies at https:/go.microsoft.com/fwlink/?LinkID=135170");
+              exit 1;
+            }
+
             try
             {
               Invoke-Command @invokeArgs
             }
             catch
             {
-              Write-Error $_.Exception
-              exit 1
+              $Host.UI.WriteErrorLine("[$($_.FullyQualifiedErrorId)] Exception $($_.InvocationInfo.PositionMessage).`n$($_.Exception.Message)");
+              exit 1;
             }
             PS
           end
 
+          def append_ps_module_path(directory)
+            <<~PS
+            $env:PSModulePath += ";#{directory}"
+            PS
+          end
+
           def ps_task(path, arguments)
             <<~PS
             $private:tempArgs = Get-ContentAsJson (
-              $utf8.GetString([System.Convert]::FromBase64String('#{Base64.encode64(JSON.dump(arguments))}'))
+              [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String('#{Base64.encode64(JSON.dump(arguments))}'))
             )
             $allowedArgs = (Get-Command "#{path}").Parameters.Keys
             $private:taskArgs = @{}
             $private:tempArgs.Keys | ? { $allowedArgs -contains $_ } | % { $private:taskArgs[$_] = $private:tempArgs[$_] }
-            try { & "#{path}" @taskArgs } catch { Write-Error $_.Exception; exit 1 }
+            try {
+              & "#{path}" @taskArgs
+            } catch {
+              $Host.UI.WriteErrorLine("[$($_.FullyQualifiedErrorId)] Exception $($_.InvocationInfo.PositionMessage).`n$($_.Exception.Message)");
+              exit 1;
+            }
             PS
           end
 
@@ -102,151 +134,11 @@ module Bolt
             "${boltBaseDir}\\hiera\\lib;" +
             $ENV:RUBYLIB
 
-            Add-Type -AssemblyName System.ServiceModel.Web, System.Runtime.Serialization
-            $utf8 = [System.Text.Encoding]::UTF8
-
-            function Write-Stream {
-            PARAM(
-              [Parameter(Position=0)] $stream,
-              [Parameter(ValueFromPipeline=$true)] $string
-            )
-            PROCESS {
-              $bytes = $utf8.GetBytes($string)
-              $stream.Write( $bytes, 0, $bytes.Length )
-            }
-            }
-
-            function Convert-JsonToXml {
-            PARAM([Parameter(ValueFromPipeline=$true)] [string[]] $json)
-            BEGIN {
-              $mStream = New-Object System.IO.MemoryStream
-            }
-            PROCESS {
-              $json | Write-Stream -Stream $mStream
-            }
-            END {
-              $mStream.Position = 0
-              try {
-                $jsonReader = [System.Runtime.Serialization.Json.JsonReaderWriterFactory]::CreateJsonReader($mStream,[System.Xml.XmlDictionaryReaderQuotas]::Max)
-                $xml = New-Object Xml.XmlDocument
-                $xml.Load($jsonReader)
-                $xml
-              } finally {
-                $jsonReader.Close()
-                $mStream.Dispose()
-              }
-            }
-            }
-
-            Function ConvertFrom-Xml {
-            [CmdletBinding(DefaultParameterSetName="AutoType")]
-            PARAM(
-              [Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=1)] [Xml.XmlNode] $xml,
-              [Parameter(Mandatory=$true,ParameterSetName="ManualType")] [Type] $Type,
-              [Switch] $ForceType
-            )
-            PROCESS{
-              if (Get-Member -InputObject $xml -Name root) {
-                return $xml.root.Objects | ConvertFrom-Xml
-              } elseif (Get-Member -InputObject $xml -Name Objects) {
-                return $xml.Objects | ConvertFrom-Xml
-              }
-              $propbag = @{}
-              foreach ($name in Get-Member -InputObject $xml -MemberType Properties | Where-Object{$_.Name -notmatch "^(__.*|type)$"} | Select-Object -ExpandProperty name) {
-                Write-Debug "$Name Type: $($xml.$Name.type)" -Debug:$false
-                $propbag."$Name" = Convert-Properties $xml."$name"
-              }
-              if (!$Type -and $xml.HasAttribute("__type")) { $Type = $xml.__Type }
-              if ($ForceType -and $Type) {
-                try {
-                  $output = New-Object $Type -Property $propbag
-                } catch {
-                  $output = New-Object PSObject -Property $propbag
-                  $output.PsTypeNames.Insert(0, $xml.__type)
-                }
-              } elseif ($propbag.Count -ne 0) {
-                $output = New-Object PSObject -Property $propbag
-                if ($Type) {
-                  $output.PsTypeNames.Insert(0, $Type)
-                }
-              }
-              return $output
-            }
-            }
-
-            Function Convert-Properties {
-            PARAM($InputObject)
-            switch ($InputObject.type) {
-              "object" {
-                return (ConvertFrom-Xml -Xml $InputObject)
-              }
-              "string" {
-                $MightBeADate = $InputObject.get_InnerText() -as [DateTime]
-                ## Strings that are actually dates (*grumble* JSON is crap)
-                if ($MightBeADate -and $propbag."$Name" -eq $MightBeADate.ToString("G")) {
-                  return $MightBeADate
-                } else {
-                  return $InputObject.get_InnerText()
-                }
-              }
-              "number" {
-                $number = $InputObject.get_InnerText()
-                if ($number -eq ($number -as [int])) {
-                  return $number -as [int]
-                } elseif ($number -eq ($number -as [double])) {
-                  return $number -as [double]
-                } else {
-                  return $number -as [decimal]
-                }
-              }
-              "boolean" {
-                return [bool]::parse($InputObject.get_InnerText())
-              }
-              "null" {
-                return $null
-              }
-              "array" {
-                [object[]]$Items = $(foreach( $item in $InputObject.GetEnumerator() ) {
-                  Convert-Properties $item
-                })
-                return $Items
-              }
-              default {
-                return $InputObject
-              }
-            }
-            }
-
-            Function ConvertFrom-Json2 {
-            [CmdletBinding()]
-            PARAM(
-              [Parameter(ValueFromPipeline=$true,Mandatory=$true,Position=1)] [string] $InputObject,
-              [Parameter(Mandatory=$true)] [Type] $Type,
-              [Switch] $ForceType
-            )
-            PROCESS {
-              $null = $PSBoundParameters.Remove("InputObject")
-              [Xml.XmlElement]$xml = (Convert-JsonToXml $InputObject).Root
-              if ($xml) {
-                if ($xml.Objects) {
-                  $xml.Objects.Item.GetEnumerator() | ConvertFrom-Xml @PSBoundParameters
-                } elseif ($xml.Item -and $xml.Item -isnot [System.Management.Automation.PSParameterizedProperty]) {
-                  $xml.Item | ConvertFrom-Xml @PSBoundParameters
-                } else {
-                  $xml | ConvertFrom-Xml @PSBoundParameters
-                }
-              } else {
-                Write-Error "Failed to parse JSON with JsonReader" -Debug:$false
-              }
-            }
-            }
-
             function ConvertFrom-PSCustomObject
             {
             PARAM([Parameter(ValueFromPipeline = $true)] $InputObject)
             PROCESS {
               if ($null -eq $InputObject) { return $null }
-
               if ($InputObject -is [System.Collections.IEnumerable] -and $InputObject -isnot [string]) {
                 $collection = @(
                   foreach ($object in $InputObject) { ConvertFrom-PSCustomObject $object }
@@ -274,12 +166,7 @@ module Bolt
               [Parameter(Mandatory = $false)] [Text.Encoding] $Encoding = [Text.Encoding]::UTF8
             )
 
-            # using polyfill cmdlet on PS2, so pass type info
-            if ($PSVersionTable.PSVersion -lt [Version]'3.0') {
-              $Text | ConvertFrom-Json2 -Type PSObject | ConvertFrom-PSCustomObject
-            } else {
-              $Text | ConvertFrom-Json | ConvertFrom-PSCustomObject
-            }
+            $Text | ConvertFrom-Json | ConvertFrom-PSCustomObject
             }
             PS
           end